diff --git "a/checkpoint-25000/trainer_state.json" "b/checkpoint-25000/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-25000/trainer_state.json" @@ -0,0 +1,186088 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.49648488700003973, + "global_step": 25000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-07, + "loss": 10.8568, + "theoretical_loss": 20.812814784551147, + "tokens_seen": 65536 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-06, + "loss": 10.8727, + "theoretical_loss": 17.56619808733264, + "tokens_seen": 131072 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904763e-06, + "loss": 10.7356, + "theoretical_loss": 15.939474075840563, + "tokens_seen": 196608 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-06, + "loss": 10.7113, + "theoretical_loss": 14.892313738992565, + "tokens_seen": 262144 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-06, + "loss": 10.3498, + "theoretical_loss": 14.13621392076697, + "tokens_seen": 327680 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809525e-06, + "loss": 10.1826, + "theoretical_loss": 13.552558455554218, + "tokens_seen": 393216 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-06, + "loss": 10.0176, + "theoretical_loss": 13.081805984405184, + "tokens_seen": 458752 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 9.6515, + "theoretical_loss": 12.690126608487319, + "tokens_seen": 524288 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571428e-06, + "loss": 9.2598, + "theoretical_loss": 12.356589446877619, + "tokens_seen": 589824 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-06, + "loss": 9.1219, + "theoretical_loss": 12.067409590039073, + "tokens_seen": 655360 + }, + { + "epoch": 0.0, + "learning_rate": 1.0912698412698412e-05, + "loss": 9.2934, + "theoretical_loss": 11.813063214105672, + "tokens_seen": 720896 + }, + { + "epoch": 0.0, + "learning_rate": 1.1904761904761905e-05, + "loss": 9.0973, + "theoretical_loss": 11.586716191710725, + "tokens_seen": 786432 + }, + { + "epoch": 0.0, + "learning_rate": 1.2896825396825396e-05, + "loss": 9.4804, + "theoretical_loss": 11.383311123190783, + "tokens_seen": 851968 + }, + { + "epoch": 0.0, + "learning_rate": 1.3888888888888888e-05, + "loss": 9.3208, + "theoretical_loss": 11.199008685115867, + "tokens_seen": 917504 + }, + { + "epoch": 0.0, + "learning_rate": 1.4880952380952381e-05, + "loss": 9.1864, + "theoretical_loss": 11.030830900981908, + "tokens_seen": 983040 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 9.398, + "theoretical_loss": 10.876425069460945, + "tokens_seen": 1048576 + }, + { + "epoch": 0.0, + "learning_rate": 1.6865079365079364e-05, + "loss": 8.9394, + "theoretical_loss": 10.73390272306672, + "tokens_seen": 1114112 + }, + { + "epoch": 0.0, + "learning_rate": 1.7857142857142855e-05, + "loss": 9.0636, + "theoretical_loss": 10.601726859234274, + "tokens_seen": 1179648 + }, + { + "epoch": 0.0, + "learning_rate": 1.884920634920635e-05, + "loss": 9.1583, + "theoretical_loss": 10.478631155360636, + "tokens_seen": 1245184 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-05, + "loss": 9.1124, + "theoretical_loss": 10.363560926767326, + "tokens_seen": 1310720 + }, + { + "epoch": 0.0, + "learning_rate": 2.0833333333333333e-05, + "loss": 9.0087, + "theoretical_loss": 10.25562920390074, + "tokens_seen": 1376256 + }, + { + "epoch": 0.0, + "learning_rate": 2.1825396825396824e-05, + "loss": 9.0788, + "theoretical_loss": 10.154083536274014, + "tokens_seen": 1441792 + }, + { + "epoch": 0.0, + "learning_rate": 2.2817460317460315e-05, + "loss": 8.4406, + "theoretical_loss": 10.058280544736593, + "tokens_seen": 1507328 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 8.9754, + "theoretical_loss": 9.967666161844274, + "tokens_seen": 1572864 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 12267, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 8.854658126831055, + "objective/train/theoretical_loss": 9.881760109397105, + "objective/train/tokens_used": 22098400, + "theoretical_loss": 9.881760109397105, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 2.48015873015873e-05, + "loss": 8.9164, + "theoretical_loss": 9.881760109397105, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 2.5793650793650793e-05, + "loss": 7.9452, + "theoretical_loss": 9.800143574544556, + "tokens_seen": 1703936 + }, + { + "epoch": 0.0, + "learning_rate": 2.6785714285714284e-05, + "loss": 8.81, + "theoretical_loss": 9.722449329911441, + "tokens_seen": 1769472 + }, + { + "epoch": 0.0, + "learning_rate": 2.7777777777777776e-05, + "loss": 9.0741, + "theoretical_loss": 9.648353742085542, + "tokens_seen": 1835008 + }, + { + "epoch": 0.0, + "learning_rate": 2.876984126984127e-05, + "loss": 8.8256, + "theoretical_loss": 9.577570254149634, + "tokens_seen": 1900544 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904762e-05, + "loss": 8.3274, + "theoretical_loss": 9.509844029768848, + "tokens_seen": 1966080 + }, + { + "epoch": 0.0, + "learning_rate": 3.075396825396825e-05, + "loss": 8.55, + "theoretical_loss": 9.44494752063593, + "tokens_seen": 2031616 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 8.0979, + "theoretical_loss": 9.382676773914453, + "tokens_seen": 2097152 + }, + { + "epoch": 0.0, + "learning_rate": 3.273809523809524e-05, + "loss": 8.3374, + "theoretical_loss": 9.322848337237977, + "tokens_seen": 2162688 + }, + { + "epoch": 0.0, + "learning_rate": 3.373015873015873e-05, + "loss": 8.5177, + "theoretical_loss": 9.26529664966427, + "tokens_seen": 2228224 + }, + { + "epoch": 0.0, + "learning_rate": 3.472222222222222e-05, + "loss": 8.6686, + "theoretical_loss": 9.209871830448751, + "tokens_seen": 2293760 + }, + { + "epoch": 0.0, + "learning_rate": 3.571428571428571e-05, + "loss": 8.675, + "theoretical_loss": 9.156437795512288, + "tokens_seen": 2359296 + }, + { + "epoch": 0.0, + "learning_rate": 3.670634920634921e-05, + "loss": 8.4694, + "theoretical_loss": 9.104870645417346, + "tokens_seen": 2424832 + }, + { + "epoch": 0.0, + "learning_rate": 3.76984126984127e-05, + "loss": 8.2407, + "theoretical_loss": 9.05505727953773, + "tokens_seen": 2490368 + }, + { + "epoch": 0.0, + "learning_rate": 3.8690476190476195e-05, + "loss": 8.5934, + "theoretical_loss": 9.006894199647824, + "tokens_seen": 2555904 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 7.8491, + "theoretical_loss": 8.960286472913351, + "tokens_seen": 2621440 + }, + { + "epoch": 0.0, + "learning_rate": 4.067460317460318e-05, + "loss": 7.8615, + "theoretical_loss": 8.915146829644607, + "tokens_seen": 2686976 + }, + { + "epoch": 0.0, + "learning_rate": 4.1666666666666665e-05, + "loss": 7.7589, + "theoretical_loss": 8.871394875482219, + "tokens_seen": 2752512 + }, + { + "epoch": 0.0, + "learning_rate": 4.265873015873016e-05, + "loss": 8.0014, + "theoretical_loss": 8.828956401157495, + "tokens_seen": 2818048 + }, + { + "epoch": 0.0, + "learning_rate": 4.365079365079365e-05, + "loss": 7.8309, + "theoretical_loss": 8.787762775782408, + "tokens_seen": 2883584 + }, + { + "epoch": 0.0, + "learning_rate": 4.464285714285714e-05, + "loss": 8.259, + "theoretical_loss": 8.747750411915451, + "tokens_seen": 2949120 + }, + { + "epoch": 0.0, + "learning_rate": 4.563492063492063e-05, + "loss": 8.1107, + "theoretical_loss": 8.708860292524829, + "tokens_seen": 3014656 + }, + { + "epoch": 0.0, + "learning_rate": 4.6626984126984126e-05, + "loss": 8.2947, + "theoretical_loss": 8.671037551512843, + "tokens_seen": 3080192 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 7.764, + "theoretical_loss": 8.63423110073947, + "tokens_seen": 3145728 + }, + { + "epoch": 0.0, + "learning_rate": 4.8611111111111115e-05, + "loss": 8.1678, + "theoretical_loss": 8.59839329754032, + "tokens_seen": 3211264 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 13483, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 8.108365058898926, + "objective/train/theoretical_loss": 8.563479647615063, + "objective/train/tokens_used": 23736800, + "theoretical_loss": 8.563479647615063, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-05, + "loss": 7.5801, + "theoretical_loss": 8.563479647615063, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 5.05952380952381e-05, + "loss": 7.8434, + "theoretical_loss": 8.52944853889911, + "tokens_seen": 3342336 + }, + { + "epoch": 0.0, + "learning_rate": 5.1587301587301586e-05, + "loss": 7.8492, + "theoretical_loss": 8.496261002649998, + "tokens_seen": 3407872 + }, + { + "epoch": 0.0, + "learning_rate": 5.257936507936508e-05, + "loss": 7.8312, + "theoretical_loss": 8.463880498501183, + "tokens_seen": 3473408 + }, + { + "epoch": 0.0, + "learning_rate": 5.357142857142857e-05, + "loss": 7.3015, + "theoretical_loss": 8.432272720676774, + "tokens_seen": 3538944 + }, + { + "epoch": 0.0, + "learning_rate": 5.4563492063492063e-05, + "loss": 7.9474, + "theoretical_loss": 8.401405422934712, + "tokens_seen": 3604480 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555555e-05, + "loss": 7.6148, + "theoretical_loss": 8.371248260124204, + "tokens_seen": 3670016 + }, + { + "epoch": 0.0, + "learning_rate": 5.6547619047619046e-05, + "loss": 7.5108, + "theoretical_loss": 8.341772644515071, + "tokens_seen": 3735552 + }, + { + "epoch": 0.0, + "learning_rate": 5.753968253968254e-05, + "loss": 7.6168, + "theoretical_loss": 8.312951615289325, + "tokens_seen": 3801088 + }, + { + "epoch": 0.0, + "learning_rate": 5.8531746031746036e-05, + "loss": 7.8436, + "theoretical_loss": 8.284759719785177, + "tokens_seen": 3866624 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809524e-05, + "loss": 7.0651, + "theoretical_loss": 8.257172905255858, + "tokens_seen": 3932160 + }, + { + "epoch": 0.0, + "learning_rate": 6.051587301587302e-05, + "loss": 7.4859, + "theoretical_loss": 8.23016842005411, + "tokens_seen": 3997696 + }, + { + "epoch": 0.0, + "learning_rate": 6.15079365079365e-05, + "loss": 7.531, + "theoretical_loss": 8.203724723281965, + "tokens_seen": 4063232 + }, + { + "epoch": 0.0, + "learning_rate": 6.25e-05, + "loss": 7.5815, + "theoretical_loss": 8.177821402057042, + "tokens_seen": 4128768 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 7.2471, + "theoretical_loss": 8.152439095643611, + "tokens_seen": 4194304 + }, + { + "epoch": 0.0, + "learning_rate": 6.448412698412699e-05, + "loss": 7.4488, + "theoretical_loss": 8.127559425781495, + "tokens_seen": 4259840 + }, + { + "epoch": 0.0, + "learning_rate": 6.547619047619048e-05, + "loss": 7.4517, + "theoretical_loss": 8.103164932619705, + "tokens_seen": 4325376 + }, + { + "epoch": 0.0, + "learning_rate": 6.646825396825397e-05, + "loss": 7.1039, + "theoretical_loss": 8.079239015726635, + "tokens_seen": 4390912 + }, + { + "epoch": 0.0, + "learning_rate": 6.746031746031745e-05, + "loss": 7.1279, + "theoretical_loss": 8.055765879705412, + "tokens_seen": 4456448 + }, + { + "epoch": 0.0, + "learning_rate": 6.845238095238096e-05, + "loss": 7.3354, + "theoretical_loss": 8.032730483993003, + "tokens_seen": 4521984 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-05, + "loss": 7.1485, + "theoretical_loss": 8.01011849646583, + "tokens_seen": 4587520 + }, + { + "epoch": 0.0, + "learning_rate": 7.043650793650793e-05, + "loss": 6.9851, + "theoretical_loss": 7.987916250513374, + "tokens_seen": 4653056 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857142e-05, + "loss": 6.9432, + "theoretical_loss": 7.966110705275796, + "tokens_seen": 4718592 + }, + { + "epoch": 0.0, + "learning_rate": 7.242063492063492e-05, + "loss": 6.9933, + "theoretical_loss": 7.944689408771984, + "tokens_seen": 4784128 + }, + { + "epoch": 0.0, + "learning_rate": 7.341269841269842e-05, + "loss": 7.0325, + "theoretical_loss": 7.923640463671514, + "tokens_seen": 4849664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 14033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.057826519012451, + "objective/train/theoretical_loss": 7.902952495488062, + "objective/train/tokens_used": 25375200, + "theoretical_loss": 7.902952495488062, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 7.440476190476191e-05, + "loss": 6.7791, + "theoretical_loss": 7.902952495488062, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 7.53968253968254e-05, + "loss": 6.7978, + "theoretical_loss": 7.882614622993199, + "tokens_seen": 4980736 + }, + { + "epoch": 0.0, + "learning_rate": 7.63888888888889e-05, + "loss": 6.8558, + "theoretical_loss": 7.8626164306686235, + "tokens_seen": 5046272 + }, + { + "epoch": 0.0, + "learning_rate": 7.738095238095239e-05, + "loss": 6.9141, + "theoretical_loss": 7.842947943031932, + "tokens_seen": 5111808 + }, + { + "epoch": 0.0, + "learning_rate": 7.837301587301588e-05, + "loss": 6.9255, + "theoretical_loss": 7.8235996006863076, + "tokens_seen": 5177344 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 6.7895, + "theoretical_loss": 7.8045622379581605, + "tokens_seen": 5242880 + }, + { + "epoch": 0.0, + "learning_rate": 8.035714285714287e-05, + "loss": 6.7059, + "theoretical_loss": 7.785827061999068, + "tokens_seen": 5308416 + }, + { + "epoch": 0.0, + "learning_rate": 8.134920634920635e-05, + "loss": 6.815, + "theoretical_loss": 7.767385633239359, + "tokens_seen": 5373952 + }, + { + "epoch": 0.0, + "learning_rate": 8.234126984126984e-05, + "loss": 6.8445, + "theoretical_loss": 7.749229847090614, + "tokens_seen": 5439488 + }, + { + "epoch": 0.0, + "learning_rate": 8.333333333333333e-05, + "loss": 6.7849, + "theoretical_loss": 7.731351916803314, + "tokens_seen": 5505024 + }, + { + "epoch": 0.0, + "learning_rate": 8.432539682539683e-05, + "loss": 6.5349, + "theoretical_loss": 7.7137443573939155, + "tokens_seen": 5570560 + }, + { + "epoch": 0.0, + "learning_rate": 8.531746031746032e-05, + "loss": 6.5103, + "theoretical_loss": 7.696399970562929, + "tokens_seen": 5636096 + }, + { + "epoch": 0.0, + "learning_rate": 8.630952380952381e-05, + "loss": 6.9307, + "theoretical_loss": 7.679311830532177, + "tokens_seen": 5701632 + }, + { + "epoch": 0.0, + "learning_rate": 8.73015873015873e-05, + "loss": 6.8115, + "theoretical_loss": 7.662473270735324, + "tokens_seen": 5767168 + }, + { + "epoch": 0.0, + "learning_rate": 8.82936507936508e-05, + "loss": 6.6601, + "theoretical_loss": 7.645877871301275, + "tokens_seen": 5832704 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571429e-05, + "loss": 6.2238, + "theoretical_loss": 7.6295194472748555, + "tokens_seen": 5898240 + }, + { + "epoch": 0.0, + "learning_rate": 9.027777777777777e-05, + "loss": 6.6176, + "theoretical_loss": 7.613392037523692, + "tokens_seen": 5963776 + }, + { + "epoch": 0.0, + "learning_rate": 9.126984126984126e-05, + "loss": 6.5932, + "theoretical_loss": 7.597489894284275, + "tokens_seen": 6029312 + }, + { + "epoch": 0.0, + "learning_rate": 9.226190476190476e-05, + "loss": 6.51, + "theoretical_loss": 7.581807473303883, + "tokens_seen": 6094848 + }, + { + "epoch": 0.0, + "learning_rate": 9.325396825396825e-05, + "loss": 6.3768, + "theoretical_loss": 7.566339424538366, + "tokens_seen": 6160384 + }, + { + "epoch": 0.0, + "learning_rate": 9.424603174603175e-05, + "loss": 6.5465, + "theoretical_loss": 7.551080583368945, + "tokens_seen": 6225920 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 6.8931, + "theoretical_loss": 7.536025962303915, + "tokens_seen": 6291456 + }, + { + "epoch": 0.0, + "learning_rate": 9.623015873015874e-05, + "loss": 6.5266, + "theoretical_loss": 7.521170743133757, + "tokens_seen": 6356992 + }, + { + "epoch": 0.0, + "learning_rate": 9.722222222222223e-05, + "loss": 6.4856, + "theoretical_loss": 7.506510269510493, + "tokens_seen": 6422528 + }, + { + "epoch": 0.0, + "learning_rate": 9.821428571428572e-05, + "loss": 6.4763, + "theoretical_loss": 7.492040039924245, + "tokens_seen": 6488064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 15357, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.699707984924316, + "objective/train/theoretical_loss": 7.4777557010520255, + "objective/train/tokens_used": 27013600, + "theoretical_loss": 7.4777557010520255, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-05, + "loss": 6.6532, + "theoretical_loss": 7.4777557010520255, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010019841269841271, + "loss": 6.4, + "theoretical_loss": 7.463653041455457, + "tokens_seen": 6619136 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001011904761904762, + "loss": 6.2462, + "theoretical_loss": 7.449727985605911, + "tokens_seen": 6684672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010218253968253968, + "loss": 6.6399, + "theoretical_loss": 7.435976588217015, + "tokens_seen": 6750208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460317, + "loss": 6.1887, + "theoretical_loss": 7.4223950288659, + "tokens_seen": 6815744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010416666666666667, + "loss": 6.4129, + "theoretical_loss": 7.40897960688587, + "tokens_seen": 6881280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010515873015873016, + "loss": 6.2296, + "theoretical_loss": 7.395726736514341, + "tokens_seen": 6946816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010615079365079365, + "loss": 6.3505, + "theoretical_loss": 7.3826329422810275, + "tokens_seen": 7012352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010714285714285714, + "loss": 6.5043, + "theoretical_loss": 7.369694854622368, + "tokens_seen": 7077888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010813492063492064, + "loss": 6.3689, + "theoretical_loss": 7.3569092057090835, + "tokens_seen": 7143424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010912698412698413, + "loss": 6.5583, + "theoretical_loss": 7.3442728254746825, + "tokens_seen": 7208960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011011904761904761, + "loss": 6.3757, + "theoretical_loss": 7.3317826378335145, + "tokens_seen": 7274496 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001111111111111111, + "loss": 6.3164, + "theoretical_loss": 7.319435657077673, + "tokens_seen": 7340032 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001121031746031746, + "loss": 6.205, + "theoretical_loss": 7.307228984442819, + "tokens_seen": 7405568 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011309523809523809, + "loss": 6.4975, + "theoretical_loss": 7.29515980483356, + "tokens_seen": 7471104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011408730158730158, + "loss": 6.087, + "theoretical_loss": 7.283225383699648, + "tokens_seen": 7536640 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011507936507936508, + "loss": 6.0785, + "theoretical_loss": 7.271423064054828, + "tokens_seen": 7602176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011607142857142858, + "loss": 6.4949, + "theoretical_loss": 7.259750263630618, + "tokens_seen": 7667712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011706349206349207, + "loss": 6.2155, + "theoretical_loss": 7.248204472157866, + "tokens_seen": 7733248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011805555555555556, + "loss": 6.0229, + "theoretical_loss": 7.236783248769257, + "tokens_seen": 7798784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 6.6244, + "theoretical_loss": 7.225484219516492, + "tokens_seen": 7864320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012003968253968255, + "loss": 5.9387, + "theoretical_loss": 7.2143050749961155, + "tokens_seen": 7929856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012103174603174604, + "loss": 6.2842, + "theoretical_loss": 7.2032435680783955, + "tokens_seen": 7995392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012202380952380953, + "loss": 6.2047, + "theoretical_loss": 7.19229751173401, + "tokens_seen": 8060928 + }, + { + "epoch": 0.0, + "learning_rate": 0.000123015873015873, + "loss": 6.2571, + "theoretical_loss": 7.181464776953511, + "tokens_seen": 8126464 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 16711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.2001729011535645, + "objective/train/theoretical_loss": 7.1707432907549595, + "objective/train/tokens_used": 28652000, + "theoretical_loss": 7.1707432907549595, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001240079365079365, + "loss": 6.0533, + "theoretical_loss": 7.1707432907549595, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000125, + "loss": 6.0526, + "theoretical_loss": 7.160131034275267, + "tokens_seen": 8257536 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001259920634920635, + "loss": 6.0532, + "theoretical_loss": 7.149626040941133, + "tokens_seen": 8323072 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 5.9719, + "theoretical_loss": 7.139226394715633, + "tokens_seen": 8388608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012797619047619048, + "loss": 6.0403, + "theoretical_loss": 7.12893022841679, + "tokens_seen": 8454144 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012896825396825398, + "loss": 5.7847, + "theoretical_loss": 7.118735722104612, + "tokens_seen": 8519680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012996031746031748, + "loss": 5.9094, + "theoretical_loss": 7.10864110153334, + "tokens_seen": 8585216 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013095238095238096, + "loss": 6.0872, + "theoretical_loss": 7.098644636665766, + "tokens_seen": 8650752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013194444444444446, + "loss": 6.0185, + "theoretical_loss": 7.088744640246688, + "tokens_seen": 8716288 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013293650793650793, + "loss": 6.0768, + "theoretical_loss": 7.078939466432745, + "tokens_seen": 8781824 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013392857142857144, + "loss": 6.3022, + "theoretical_loss": 7.069227509475962, + "tokens_seen": 8847360 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001349206349206349, + "loss": 6.0881, + "theoretical_loss": 7.059607202458563, + "tokens_seen": 8912896 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001359126984126984, + "loss": 5.7381, + "theoretical_loss": 7.0500770160766635, + "tokens_seen": 8978432 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001369047619047619, + "loss": 5.9916, + "theoretical_loss": 7.04063545747062, + "tokens_seen": 9043968 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013789682539682541, + "loss": 6.0983, + "theoretical_loss": 7.031281069099929, + "tokens_seen": 9109504 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001388888888888889, + "loss": 6.0662, + "theoretical_loss": 7.022012427660673, + "tokens_seen": 9175040 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001398809523809524, + "loss": 5.7078, + "theoretical_loss": 7.012828143043604, + "tokens_seen": 9240576 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014087301587301586, + "loss": 6.2455, + "theoretical_loss": 7.003726857331067, + "tokens_seen": 9306112 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014186507936507937, + "loss": 5.9609, + "theoretical_loss": 6.994707243831053, + "tokens_seen": 9371648 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714284, + "loss": 5.9423, + "theoretical_loss": 6.985768006146776, + "tokens_seen": 9437184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014384920634920634, + "loss": 6.0329, + "theoretical_loss": 6.976907877280185, + "tokens_seen": 9502720 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014484126984126984, + "loss": 5.7799, + "theoretical_loss": 6.96812561876801, + "tokens_seen": 9568256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014583333333333335, + "loss": 5.9063, + "theoretical_loss": 6.95942001984889, + "tokens_seen": 9633792 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014682539682539685, + "loss": 5.9317, + "theoretical_loss": 6.950789896660304, + "tokens_seen": 9699328 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014781746031746032, + "loss": 5.8925, + "theoretical_loss": 6.942234091464025, + "tokens_seen": 9764864 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 17166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.479184627532959, + "objective/train/theoretical_loss": 6.933751471898896, + "objective/train/tokens_used": 30290400, + "theoretical_loss": 6.933751471898896, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014880952380952382, + "loss": 6.2539, + "theoretical_loss": 6.933751471898896, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001498015873015873, + "loss": 5.996, + "theoretical_loss": 6.925340930259813, + "tokens_seen": 9895936 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 5.8197, + "theoretical_loss": 6.917001382801793, + "tokens_seen": 9961472 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015178571428571427, + "loss": 6.0534, + "theoretical_loss": 6.908731769068142, + "tokens_seen": 10027008 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001527777777777778, + "loss": 5.7838, + "theoretical_loss": 6.900531051241684, + "tokens_seen": 10092544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015376984126984128, + "loss": 5.7799, + "theoretical_loss": 6.892398213518156, + "tokens_seen": 10158080 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015476190476190478, + "loss": 6.1056, + "theoretical_loss": 6.884332261500866, + "tokens_seen": 10223616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015575396825396825, + "loss": 5.5432, + "theoretical_loss": 6.876332221615746, + "tokens_seen": 10289152 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015674603174603175, + "loss": 5.8572, + "theoretical_loss": 6.8683971405459925, + "tokens_seen": 10354688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015773809523809523, + "loss": 6.226, + "theoretical_loss": 6.860526084685546, + "tokens_seen": 10420224 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 5.7117, + "theoretical_loss": 6.852718139610625, + "tokens_seen": 10485760 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001597222222222222, + "loss": 5.6684, + "theoretical_loss": 6.844972409568637, + "tokens_seen": 10551296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016071428571428573, + "loss": 5.8166, + "theoretical_loss": 6.8372880169837895, + "tokens_seen": 10616832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001617063492063492, + "loss": 5.971, + "theoretical_loss": 6.829664101978745, + "tokens_seen": 10682368 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001626984126984127, + "loss": 5.7343, + "theoretical_loss": 6.8220998219117135, + "tokens_seen": 10747904 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016369047619047618, + "loss": 5.8061, + "theoretical_loss": 6.81459435092839, + "tokens_seen": 10813440 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016468253968253969, + "loss": 6.0029, + "theoretical_loss": 6.807146879528176, + "tokens_seen": 10878976 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016567460317460316, + "loss": 5.5367, + "theoretical_loss": 6.79975661414414, + "tokens_seen": 10944512 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016666666666666666, + "loss": 5.6256, + "theoretical_loss": 6.79242277673622, + "tokens_seen": 11010048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016765873015873016, + "loss": 5.7868, + "theoretical_loss": 6.785144604397143, + "tokens_seen": 11075584 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016865079365079366, + "loss": 5.6046, + "theoretical_loss": 6.777921348970633, + "tokens_seen": 11141120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016964285714285717, + "loss": 5.9753, + "theoretical_loss": 6.770752276681419, + "tokens_seen": 11206656 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017063492063492064, + "loss": 5.9022, + "theoretical_loss": 6.76363666777662, + "tokens_seen": 11272192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017162698412698414, + "loss": 5.8921, + "theoretical_loss": 6.756573816178118, + "tokens_seen": 11337728 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017261904761904762, + "loss": 5.9722, + "theoretical_loss": 6.749563029145481, + "tokens_seen": 11403264 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 18369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.672413349151611, + "objective/train/theoretical_loss": 6.742603626949085, + "objective/train/tokens_used": 31928800, + "theoretical_loss": 6.742603626949085, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017361111111111112, + "loss": 5.4506, + "theoretical_loss": 6.742603626949085, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001746031746031746, + "loss": 6.0078, + "theoretical_loss": 6.735694942553071, + "tokens_seen": 11534336 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001755952380952381, + "loss": 5.8091, + "theoretical_loss": 6.728836321307757, + "tokens_seen": 11599872 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001765873015873016, + "loss": 5.6113, + "theoretical_loss": 6.722027120651221, + "tokens_seen": 11665408 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001775793650793651, + "loss": 5.6758, + "theoretical_loss": 6.715266709819684, + "tokens_seen": 11730944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017857142857142857, + "loss": 5.6378, + "theoretical_loss": 6.708554469566408, + "tokens_seen": 11796480 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017956349206349207, + "loss": 5.7411, + "theoretical_loss": 6.701889791888819, + "tokens_seen": 11862016 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018055555555555555, + "loss": 5.8311, + "theoretical_loss": 6.695272079763555, + "tokens_seen": 11927552 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018154761904761905, + "loss": 5.2846, + "theoretical_loss": 6.688700746889175, + "tokens_seen": 11993088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968252, + "loss": 5.7687, + "theoretical_loss": 6.682175217436269, + "tokens_seen": 12058624 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018353174603174602, + "loss": 5.7687, + "theoretical_loss": 6.675694925804711, + "tokens_seen": 12124160 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018452380952380953, + "loss": 5.6209, + "theoretical_loss": 6.66925931638781, + "tokens_seen": 12189696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018551587301587303, + "loss": 5.9193, + "theoretical_loss": 6.662867843343154, + "tokens_seen": 12255232 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001865079365079365, + "loss": 5.6754, + "theoretical_loss": 6.6565199703698745, + "tokens_seen": 12320768 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001875, + "loss": 5.8597, + "theoretical_loss": 6.6502151704921655, + "tokens_seen": 12386304 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001884920634920635, + "loss": 5.709, + "theoretical_loss": 6.643952925848826, + "tokens_seen": 12451840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018948412698412698, + "loss": 5.7054, + "theoretical_loss": 6.637732727488622, + "tokens_seen": 12517376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 5.5972, + "theoretical_loss": 6.6315540751713, + "tokens_seen": 12582912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019146825396825398, + "loss": 5.5887, + "theoretical_loss": 6.625416477174044, + "tokens_seen": 12648448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019246031746031748, + "loss": 5.7082, + "theoretical_loss": 6.619319450103218, + "tokens_seen": 12713984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019345238095238096, + "loss": 5.5405, + "theoretical_loss": 6.613262518711206, + "tokens_seen": 12779520 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019444444444444446, + "loss": 5.5806, + "theoretical_loss": 6.607245215718208, + "tokens_seen": 12845056 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019543650793650793, + "loss": 5.4612, + "theoretical_loss": 6.601267081638806, + "tokens_seen": 12910592 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019642857142857144, + "loss": 5.6405, + "theoretical_loss": 6.595327664613176, + "tokens_seen": 12976128 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001974206349206349, + "loss": 5.3968, + "theoretical_loss": 6.58942652024278, + "tokens_seen": 13041664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 19017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.398904323577881, + "objective/train/theoretical_loss": 6.583563211430409, + "objective/train/tokens_used": 33567200, + "theoretical_loss": 6.583563211430409, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001984126984126984, + "loss": 5.5392, + "theoretical_loss": 6.583563211430409, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019940476190476191, + "loss": 5.3995, + "theoretical_loss": 6.577737308224426, + "tokens_seen": 13172736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020039682539682542, + "loss": 5.5665, + "theoretical_loss": 6.571948387667094, + "tokens_seen": 13238272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002013888888888889, + "loss": 5.5745, + "theoretical_loss": 6.566196033646859, + "tokens_seen": 13303808 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002023809523809524, + "loss": 5.4756, + "theoretical_loss": 6.560479836754459, + "tokens_seen": 13369344 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020337301587301587, + "loss": 5.7895, + "theoretical_loss": 6.554799394142741, + "tokens_seen": 13434880 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020436507936507937, + "loss": 5.3965, + "theoretical_loss": 6.549154309390087, + "tokens_seen": 13500416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020535714285714284, + "loss": 5.494, + "theoretical_loss": 6.5435441923673245, + "tokens_seen": 13565952 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920634, + "loss": 5.5559, + "theoretical_loss": 6.537968659108021, + "tokens_seen": 13631488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020734126984126985, + "loss": 5.9125, + "theoretical_loss": 6.532427331682063, + "tokens_seen": 13697024 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020833333333333335, + "loss": 5.5235, + "theoretical_loss": 6.5269198380724145, + "tokens_seen": 13762560 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020932539682539685, + "loss": 5.6307, + "theoretical_loss": 6.5214458120549725, + "tokens_seen": 13828096 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021031746031746032, + "loss": 5.3661, + "theoretical_loss": 6.516004893081412, + "tokens_seen": 13893632 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021130952380952382, + "loss": 5.2823, + "theoretical_loss": 6.510596726164945, + "tokens_seen": 13959168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002123015873015873, + "loss": 5.4241, + "theoretical_loss": 6.505220961768906, + "tokens_seen": 14024704 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002132936507936508, + "loss": 5.3679, + "theoretical_loss": 6.499877255698063, + "tokens_seen": 14090240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021428571428571427, + "loss": 5.2005, + "theoretical_loss": 6.494565268992613, + "tokens_seen": 14155776 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002152777777777778, + "loss": 5.6155, + "theoretical_loss": 6.489284667824741, + "tokens_seen": 14221312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021626984126984128, + "loss": 5.6027, + "theoretical_loss": 6.484035123397694, + "tokens_seen": 14286848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021726190476190478, + "loss": 5.5959, + "theoretical_loss": 6.478816311847305, + "tokens_seen": 14352384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021825396825396825, + "loss": 5.7842, + "theoretical_loss": 6.473627914145865, + "tokens_seen": 14417920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021924603174603176, + "loss": 5.4174, + "theoretical_loss": 6.468469616008303, + "tokens_seen": 14483456 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022023809523809523, + "loss": 5.4423, + "theoretical_loss": 6.463341107800611, + "tokens_seen": 14548992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022123015873015873, + "loss": 5.2413, + "theoretical_loss": 6.458242084450426, + "tokens_seen": 14614528 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002222222222222222, + "loss": 5.4841, + "theoretical_loss": 6.453172245359726, + "tokens_seen": 14680064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 20187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.4210638999938965, + "objective/train/theoretical_loss": 6.448131294319589, + "objective/train/tokens_used": 35205600, + "theoretical_loss": 6.448131294319589, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022321428571428573, + "loss": 5.318, + "theoretical_loss": 6.448131294319589, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002242063492063492, + "loss": 5.5298, + "theoretical_loss": 6.443118939426935, + "tokens_seen": 14811136 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002251984126984127, + "loss": 5.5104, + "theoretical_loss": 6.438134893003209, + "tokens_seen": 14876672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022619047619047618, + "loss": 5.4089, + "theoretical_loss": 6.43317887151496, + "tokens_seen": 14942208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022718253968253969, + "loss": 5.5783, + "theoretical_loss": 6.428250595496234, + "tokens_seen": 15007744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022817460317460316, + "loss": 5.4029, + "theoretical_loss": 6.4233497894727805, + "tokens_seen": 15073280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022916666666666666, + "loss": 5.6043, + "theoretical_loss": 6.418476181887964, + "tokens_seen": 15138816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 5.5123, + "theoretical_loss": 6.413629505030386, + "tokens_seen": 15204352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023115079365079367, + "loss": 5.5084, + "theoretical_loss": 6.408809494963135, + "tokens_seen": 15269888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023214285714285717, + "loss": 5.269, + "theoretical_loss": 6.404015891454651, + "tokens_seen": 15335424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023313492063492064, + "loss": 5.4379, + "theoretical_loss": 6.399248437911128, + "tokens_seen": 15400960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023412698412698414, + "loss": 5.4346, + "theoretical_loss": 6.394506881310447, + "tokens_seen": 15466496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023511904761904762, + "loss": 5.7191, + "theoretical_loss": 6.38979097213757, + "tokens_seen": 15532032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023611111111111112, + "loss": 5.5484, + "theoretical_loss": 6.385100464321383, + "tokens_seen": 15597568 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002371031746031746, + "loss": 5.4379, + "theoretical_loss": 6.380435115172919, + "tokens_seen": 15663104 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 5.4785, + "theoretical_loss": 6.375794685324962, + "tokens_seen": 15728640 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002390873015873016, + "loss": 5.2088, + "theoretical_loss": 6.371178938672962, + "tokens_seen": 15794176 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002400793650793651, + "loss": 5.5076, + "theoretical_loss": 6.366587642317243, + "tokens_seen": 15859712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024107142857142857, + "loss": 5.6351, + "theoretical_loss": 6.362020566506475, + "tokens_seen": 15925248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024206349206349207, + "loss": 5.6029, + "theoretical_loss": 6.3574774845823665, + "tokens_seen": 15990784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024305555555555555, + "loss": 5.1969, + "theoretical_loss": 6.352958172925549, + "tokens_seen": 16056320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024404761904761905, + "loss": 5.285, + "theoretical_loss": 6.348462410902625, + "tokens_seen": 16121856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024503968253968255, + "loss": 5.6805, + "theoretical_loss": 6.343989980814362, + "tokens_seen": 16187392 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246031746031746, + "loss": 5.5855, + "theoretical_loss": 6.339540667844965, + "tokens_seen": 16252928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024702380952380955, + "loss": 5.3559, + "theoretical_loss": 6.335114260012455, + "tokens_seen": 16318464 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 20737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.463630199432373, + "objective/train/theoretical_loss": 6.330710548120079, + "objective/train/tokens_used": 36844000, + "theoretical_loss": 6.330710548120079, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248015873015873, + "loss": 5.515, + "theoretical_loss": 6.330710548120079, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002490079365079365, + "loss": 5.3596, + "theoretical_loss": 6.326329325708746, + "tokens_seen": 16449536 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 5.249, + "theoretical_loss": 6.321970389010465, + "tokens_seen": 16515072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002509920634920635, + "loss": 5.3675, + "theoretical_loss": 6.317633536902759, + "tokens_seen": 16580608 + }, + { + "epoch": 0.01, + "learning_rate": 0.000251984126984127, + "loss": 5.0919, + "theoretical_loss": 6.313318570864016, + "tokens_seen": 16646144 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025297619047619046, + "loss": 5.1228, + "theoretical_loss": 6.30902529492978, + "tokens_seen": 16711680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025396825396825396, + "loss": 5.4573, + "theoretical_loss": 6.304753515649935, + "tokens_seen": 16777216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025496031746031746, + "loss": 5.1499, + "theoretical_loss": 6.30050304204677, + "tokens_seen": 16842752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025595238095238096, + "loss": 5.6432, + "theoretical_loss": 6.296273685573913, + "tokens_seen": 16908288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002569444444444444, + "loss": 5.439, + "theoretical_loss": 6.292065260076094, + "tokens_seen": 16973824 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025793650793650796, + "loss": 5.2959, + "theoretical_loss": 6.287877581749726, + "tokens_seen": 17039360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025892857142857146, + "loss": 5.3502, + "theoretical_loss": 6.2837104691042915, + "tokens_seen": 17104896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025992063492063497, + "loss": 5.5134, + "theoretical_loss": 6.279563742924502, + "tokens_seen": 17170432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002609126984126984, + "loss": 5.4871, + "theoretical_loss": 6.275437226233224, + "tokens_seen": 17235968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002619047619047619, + "loss": 5.3132, + "theoretical_loss": 6.271330744255137, + "tokens_seen": 17301504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002628968253968254, + "loss": 5.1796, + "theoretical_loss": 6.267244124381133, + "tokens_seen": 17367040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002638888888888889, + "loss": 5.2656, + "theoretical_loss": 6.2631771961334035, + "tokens_seen": 17432576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026488095238095237, + "loss": 5.5896, + "theoretical_loss": 6.259129791131242, + "tokens_seen": 17498112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026587301587301587, + "loss": 5.14, + "theoretical_loss": 6.255101743057493, + "tokens_seen": 17563648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026686507936507937, + "loss": 5.3606, + "theoretical_loss": 6.251092887625685, + "tokens_seen": 17629184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026785714285714287, + "loss": 5.487, + "theoretical_loss": 6.247103062547796, + "tokens_seen": 17694720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002688492063492063, + "loss": 5.2887, + "theoretical_loss": 6.243132107502647, + "tokens_seen": 17760256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002698412698412698, + "loss": 5.4254, + "theoretical_loss": 6.239179864104911, + "tokens_seen": 17825792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002708333333333333, + "loss": 5.4668, + "theoretical_loss": 6.235246175874727, + "tokens_seen": 17891328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002718253968253968, + "loss": 5.3584, + "theoretical_loss": 6.231330888207894, + "tokens_seen": 17956864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 21782, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.119491100311279, + "objective/train/theoretical_loss": 6.227433848346639, + "objective/train/tokens_used": 38482400, + "theoretical_loss": 6.227433848346639, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002728174603174603, + "loss": 5.1069, + "theoretical_loss": 6.227433848346639, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002738095238095238, + "loss": 5.1568, + "theoretical_loss": 6.22355490535095, + "tokens_seen": 18087936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002748015873015873, + "loss": 5.4409, + "theoretical_loss": 6.219693910070452, + "tokens_seen": 18153472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027579365079365083, + "loss": 5.1939, + "theoretical_loss": 6.215850715116817, + "tokens_seen": 18219008 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027678571428571433, + "loss": 5.2274, + "theoretical_loss": 6.212025174836697, + "tokens_seen": 18284544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002777777777777778, + "loss": 5.5299, + "theoretical_loss": 6.208217145285173, + "tokens_seen": 18350080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002787698412698413, + "loss": 5.4441, + "theoretical_loss": 6.204426484199696, + "tokens_seen": 18415616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002797619047619048, + "loss": 5.3135, + "theoretical_loss": 6.200653050974525, + "tokens_seen": 18481152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002807539682539683, + "loss": 5.189, + "theoretical_loss": 6.196896706635635, + "tokens_seen": 18546688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028174603174603173, + "loss": 5.0786, + "theoretical_loss": 6.1931573138160925, + "tokens_seen": 18612224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028273809523809523, + "loss": 5.0434, + "theoretical_loss": 6.189434736731897, + "tokens_seen": 18677760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028373015873015873, + "loss": 5.0132, + "theoretical_loss": 6.185728841158257, + "tokens_seen": 18743296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028472222222222223, + "loss": 5.1159, + "theoretical_loss": 6.182039494406309, + "tokens_seen": 18808832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002857142857142857, + "loss": 5.1786, + "theoretical_loss": 6.178366565300266, + "tokens_seen": 18874368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002867063492063492, + "loss": 5.1947, + "theoretical_loss": 6.174709924154978, + "tokens_seen": 18939904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002876984126984127, + "loss": 5.2396, + "theoretical_loss": 6.171069442753909, + "tokens_seen": 19005440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002886904761904762, + "loss": 5.3947, + "theoretical_loss": 6.1674449943275045, + "tokens_seen": 19070976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002896825396825397, + "loss": 4.8732, + "theoretical_loss": 6.16383645353196, + "tokens_seen": 19136512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002906746031746032, + "loss": 5.3178, + "theoretical_loss": 6.160243696428367, + "tokens_seen": 19202048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002916666666666667, + "loss": 5.1441, + "theoretical_loss": 6.156666600462238, + "tokens_seen": 19267584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002926587301587302, + "loss": 5.2695, + "theoretical_loss": 6.153105044443393, + "tokens_seen": 19333120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002936507936507937, + "loss": 5.3181, + "theoretical_loss": 6.149558908526206, + "tokens_seen": 19398656 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029464285714285714, + "loss": 5.2188, + "theoretical_loss": 6.146028074190217, + "tokens_seen": 19464192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029563492063492064, + "loss": 5.1652, + "theoretical_loss": 6.14251242422106, + "tokens_seen": 19529728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029662698412698414, + "loss": 5.3475, + "theoretical_loss": 6.139011842691756, + "tokens_seen": 19595264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 22379, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.661308765411377, + "objective/train/theoretical_loss": 6.135526214944321, + "objective/train/tokens_used": 40120800, + "theoretical_loss": 6.135526214944321, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029761904761904765, + "loss": 5.6218, + "theoretical_loss": 6.135526214944321, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002986111111111111, + "loss": 5.4528, + "theoretical_loss": 6.1320554275717, + "tokens_seen": 19726336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002996031746031746, + "loss": 5.1971, + "theoretical_loss": 6.128599368400017, + "tokens_seen": 19791872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003005952380952381, + "loss": 5.2823, + "theoretical_loss": 6.125157926471134, + "tokens_seen": 19857408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003015873015873016, + "loss": 5.1154, + "theoretical_loss": 6.121730992025516, + "tokens_seen": 19922944 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030257936507936505, + "loss": 4.8293, + "theoretical_loss": 6.118318456485394, + "tokens_seen": 19988480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030357142857142855, + "loss": 5.0591, + "theoretical_loss": 6.114920212438209, + "tokens_seen": 20054016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030456349206349205, + "loss": 5.2452, + "theoretical_loss": 6.111536153620355, + "tokens_seen": 20119552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003055555555555556, + "loss": 5.1158, + "theoretical_loss": 6.108166174901191, + "tokens_seen": 20185088 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030654761904761905, + "loss": 5.0927, + "theoretical_loss": 6.104810172267331, + "tokens_seen": 20250624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030753968253968255, + "loss": 5.1097, + "theoretical_loss": 6.101468042807199, + "tokens_seen": 20316160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030853174603174605, + "loss": 5.0109, + "theoretical_loss": 6.098139684695851, + "tokens_seen": 20381696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030952380952380956, + "loss": 5.2, + "theoretical_loss": 6.094824997180048, + "tokens_seen": 20447232 + }, + { + "epoch": 0.01, + "learning_rate": 0.000310515873015873, + "loss": 4.919, + "theoretical_loss": 6.091523880563589, + "tokens_seen": 20512768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003115079365079365, + "loss": 4.7726, + "theoretical_loss": 6.088236236192881, + "tokens_seen": 20578304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003125, + "loss": 5.1579, + "theoretical_loss": 6.0849619664427586, + "tokens_seen": 20643840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003134920634920635, + "loss": 5.2649, + "theoretical_loss": 6.0817009747025415, + "tokens_seen": 20709376 + }, + { + "epoch": 0.01, + "learning_rate": 0.000314484126984127, + "loss": 4.9338, + "theoretical_loss": 6.07845316536232, + "tokens_seen": 20774912 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031547619047619046, + "loss": 5.2449, + "theoretical_loss": 6.075218443799468, + "tokens_seen": 20840448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031646825396825396, + "loss": 5.2222, + "theoretical_loss": 6.071996716365382, + "tokens_seen": 20905984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031746031746031746, + "loss": 4.8122, + "theoretical_loss": 6.068787890372443, + "tokens_seen": 20971520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031845238095238096, + "loss": 4.983, + "theoretical_loss": 6.0655918740811865, + "tokens_seen": 21037056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003194444444444444, + "loss": 4.9428, + "theoretical_loss": 6.062408576687682, + "tokens_seen": 21102592 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032043650793650796, + "loss": 5.0493, + "theoretical_loss": 6.059237908311129, + "tokens_seen": 21168128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032142857142857147, + "loss": 4.8675, + "theoretical_loss": 6.056079779981644, + "tokens_seen": 21233664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 23601, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.199455738067627, + "objective/train/theoretical_loss": 6.052934103628253, + "objective/train/tokens_used": 41759200, + "theoretical_loss": 6.052934103628253, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032242063492063497, + "loss": 5.2637, + "theoretical_loss": 6.052934103628253, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003234126984126984, + "loss": 5.4325, + "theoretical_loss": 6.049800792067078, + "tokens_seen": 21364736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003244047619047619, + "loss": 4.8274, + "theoretical_loss": 6.046679758989709, + "tokens_seen": 21430272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003253968253968254, + "loss": 5.2104, + "theoretical_loss": 6.043570918951775, + "tokens_seen": 21495808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003263888888888889, + "loss": 5.1036, + "theoretical_loss": 6.040474187361681, + "tokens_seen": 21561344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032738095238095237, + "loss": 4.8811, + "theoretical_loss": 6.037389480469548, + "tokens_seen": 21626880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032837301587301587, + "loss": 5.0504, + "theoretical_loss": 6.034316715356304, + "tokens_seen": 21692416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032936507936507937, + "loss": 5.0898, + "theoretical_loss": 6.031255809922974, + "tokens_seen": 21757952 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033035714285714287, + "loss": 5.0121, + "theoretical_loss": 6.0282066828801195, + "tokens_seen": 21823488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003313492063492063, + "loss": 5.1363, + "theoretical_loss": 6.0251692537374595, + "tokens_seen": 21889024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003323412698412698, + "loss": 5.1503, + "theoretical_loss": 6.022143442793643, + "tokens_seen": 21954560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003333333333333333, + "loss": 5.1066, + "theoretical_loss": 6.0191291711261945, + "tokens_seen": 22020096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003343253968253968, + "loss": 5.0889, + "theoretical_loss": 6.016126360581609, + "tokens_seen": 22085632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003353174603174603, + "loss": 4.9077, + "theoretical_loss": 6.013134933765605, + "tokens_seen": 22151168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003363095238095238, + "loss": 5.2813, + "theoretical_loss": 6.010154814033528, + "tokens_seen": 22216704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033730158730158733, + "loss": 4.6999, + "theoretical_loss": 6.007185925480902, + "tokens_seen": 22282240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033829365079365083, + "loss": 5.0728, + "theoretical_loss": 6.0042281929341375, + "tokens_seen": 22347776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033928571428571433, + "loss": 5.1319, + "theoretical_loss": 6.001281541941363, + "tokens_seen": 22413312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003402777777777778, + "loss": 5.0915, + "theoretical_loss": 5.998345898763421, + "tokens_seen": 22478848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003412698412698413, + "loss": 4.8912, + "theoretical_loss": 5.995421190364983, + "tokens_seen": 22544384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003422619047619048, + "loss": 5.0317, + "theoretical_loss": 5.992507344405814, + "tokens_seen": 22609920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003432539682539683, + "loss": 5.1603, + "theoretical_loss": 5.9896042892321635, + "tokens_seen": 22675456 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034424603174603173, + "loss": 4.9822, + "theoretical_loss": 5.986711953868287, + "tokens_seen": 22740992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034523809523809523, + "loss": 5.3948, + "theoretical_loss": 5.983830268008107, + "tokens_seen": 22806528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034623015873015873, + "loss": 5.222, + "theoretical_loss": 5.980959162006979, + "tokens_seen": 22872064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 24859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.439995288848877, + "objective/train/theoretical_loss": 5.978098566873603, + "objective/train/tokens_used": 43397600, + "theoretical_loss": 5.978098566873603, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034722222222222224, + "loss": 4.9666, + "theoretical_loss": 5.978098566873603, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003482142857142857, + "loss": 4.7873, + "theoretical_loss": 5.975248414262053, + "tokens_seen": 23003136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003492063492063492, + "loss": 5.1429, + "theoretical_loss": 5.972408636463909, + "tokens_seen": 23068672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003501984126984127, + "loss": 4.8387, + "theoretical_loss": 5.969579166400528, + "tokens_seen": 23134208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003511904761904762, + "loss": 5.2741, + "theoretical_loss": 5.966759937615427, + "tokens_seen": 23199744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003521825396825397, + "loss": 4.9993, + "theoretical_loss": 5.96395088426676, + "tokens_seen": 23265280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003531746031746032, + "loss": 4.838, + "theoretical_loss": 5.961151941119932, + "tokens_seen": 23330816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003541666666666667, + "loss": 4.8937, + "theoretical_loss": 5.95836304354031, + "tokens_seen": 23396352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003551587301587302, + "loss": 5.1755, + "theoretical_loss": 5.9555841274860395, + "tokens_seen": 23461888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003561507936507937, + "loss": 5.0192, + "theoretical_loss": 5.952815129500973, + "tokens_seen": 23527424 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035714285714285714, + "loss": 5.1528, + "theoretical_loss": 5.950055986707699, + "tokens_seen": 23592960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035813492063492064, + "loss": 5.2473, + "theoretical_loss": 5.9473066368006755, + "tokens_seen": 23658496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035912698412698415, + "loss": 5.1431, + "theoretical_loss": 5.944567018039454, + "tokens_seen": 23724032 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036011904761904765, + "loss": 5.2078, + "theoretical_loss": 5.941837069242023, + "tokens_seen": 23789568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003611111111111111, + "loss": 5.2074, + "theoretical_loss": 5.939116729778224, + "tokens_seen": 23855104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003621031746031746, + "loss": 5.045, + "theoretical_loss": 5.936405939563279, + "tokens_seen": 23920640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003630952380952381, + "loss": 5.0272, + "theoretical_loss": 5.93370463905141, + "tokens_seen": 23986176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003640873015873016, + "loss": 5.0628, + "theoretical_loss": 5.931012769229536, + "tokens_seen": 24051712 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036507936507936505, + "loss": 4.6127, + "theoretical_loss": 5.928330271611081, + "tokens_seen": 24117248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036607142857142855, + "loss": 4.8408, + "theoretical_loss": 5.925657088229862, + "tokens_seen": 24182784 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036706349206349205, + "loss": 4.9603, + "theoretical_loss": 5.9229931616340545, + "tokens_seen": 24248320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003680555555555556, + "loss": 5.0127, + "theoretical_loss": 5.920338434880263, + "tokens_seen": 24313856 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036904761904761905, + "loss": 5.3699, + "theoretical_loss": 5.9176928515276535, + "tokens_seen": 24379392 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037003968253968255, + "loss": 4.9974, + "theoretical_loss": 5.915056355632197, + "tokens_seen": 24444928 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037103174603174606, + "loss": 4.9348, + "theoretical_loss": 5.912428891740967, + "tokens_seen": 24510464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 25411, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.414747714996338, + "objective/train/theoretical_loss": 5.90981040488653, + "objective/train/tokens_used": 45036000, + "theoretical_loss": 5.90981040488653, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037202380952380956, + "loss": 5.0376, + "theoretical_loss": 5.90981040488653, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000373015873015873, + "loss": 5.0724, + "theoretical_loss": 5.907200840581417, + "tokens_seen": 24641536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003740079365079365, + "loss": 5.2247, + "theoretical_loss": 5.904600144812672, + "tokens_seen": 24707072 + }, + { + "epoch": 0.01, + "learning_rate": 0.000375, + "loss": 5.4811, + "theoretical_loss": 5.902008264036468, + "tokens_seen": 24772608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003759920634920635, + "loss": 4.9107, + "theoretical_loss": 5.899425145172803, + "tokens_seen": 24838144 + }, + { + "epoch": 0.01, + "learning_rate": 0.000376984126984127, + "loss": 5.0933, + "theoretical_loss": 5.896850735600281, + "tokens_seen": 24903680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037797619047619046, + "loss": 5.1802, + "theoretical_loss": 5.8942849831509445, + "tokens_seen": 24969216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037896825396825396, + "loss": 5.2521, + "theoretical_loss": 5.891727836105194, + "tokens_seen": 25034752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037996031746031746, + "loss": 4.8607, + "theoretical_loss": 5.889179243186776, + "tokens_seen": 25100288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038095238095238096, + "loss": 5.0973, + "theoretical_loss": 5.886639153557828, + "tokens_seen": 25165824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003819444444444444, + "loss": 5.3397, + "theoretical_loss": 5.8841075168140105, + "tokens_seen": 25231360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038293650793650797, + "loss": 4.7777, + "theoretical_loss": 5.88158428297969, + "tokens_seen": 25296896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038392857142857147, + "loss": 4.7219, + "theoretical_loss": 5.879069402503189, + "tokens_seen": 25362432 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038492063492063497, + "loss": 5.0299, + "theoretical_loss": 5.876562826252119, + "tokens_seen": 25427968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003859126984126984, + "loss": 4.8369, + "theoretical_loss": 5.874064505508748, + "tokens_seen": 25493504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003869047619047619, + "loss": 5.0017, + "theoretical_loss": 5.871574391965453, + "tokens_seen": 25559040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003878968253968254, + "loss": 4.9705, + "theoretical_loss": 5.8690924377202265, + "tokens_seen": 25624576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003888888888888889, + "loss": 5.034, + "theoretical_loss": 5.866618595272241, + "tokens_seen": 25690112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038988095238095237, + "loss": 4.7181, + "theoretical_loss": 5.864152817517482, + "tokens_seen": 25755648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039087301587301587, + "loss": 4.9562, + "theoretical_loss": 5.8616950577444245, + "tokens_seen": 25821184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039186507936507937, + "loss": 4.9405, + "theoretical_loss": 5.859245269629783, + "tokens_seen": 25886720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003928571428571429, + "loss": 5.099, + "theoretical_loss": 5.85680340723431, + "tokens_seen": 25952256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003938492063492063, + "loss": 4.6503, + "theoretical_loss": 5.85436942499865, + "tokens_seen": 26017792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003948412698412698, + "loss": 4.8672, + "theoretical_loss": 5.851943277739253, + "tokens_seen": 26083328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003958333333333333, + "loss": 5.0201, + "theoretical_loss": 5.84952492064434, + "tokens_seen": 26148864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 26668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.337137222290039, + "objective/train/theoretical_loss": 5.847114309269919, + "objective/train/tokens_used": 46674400, + "theoretical_loss": 5.847114309269919, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003968253968253968, + "loss": 5.1138, + "theoretical_loss": 5.847114309269919, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003978174603174603, + "loss": 4.8239, + "theoretical_loss": 5.844711399535855, + "tokens_seen": 26279936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039880952380952383, + "loss": 4.7304, + "theoretical_loss": 5.842316147722, + "tokens_seen": 26345472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039980158730158733, + "loss": 5.0092, + "theoretical_loss": 5.839928510464356, + "tokens_seen": 26411008 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040079365079365083, + "loss": 4.4343, + "theoretical_loss": 5.837548444751306, + "tokens_seen": 26476544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040178571428571433, + "loss": 4.9309, + "theoretical_loss": 5.835175907919885, + "tokens_seen": 26542080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004027777777777778, + "loss": 5.0148, + "theoretical_loss": 5.832810857652097, + "tokens_seen": 26607616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004037698412698413, + "loss": 4.9709, + "theoretical_loss": 5.830453251971296, + "tokens_seen": 26673152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 5.1225, + "theoretical_loss": 5.8281030492385835, + "tokens_seen": 26738688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004057539682539683, + "loss": 4.9439, + "theoretical_loss": 5.82576020814929, + "tokens_seen": 26804224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040674603174603173, + "loss": 5.154, + "theoretical_loss": 5.823424687729469, + "tokens_seen": 26869760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040773809523809523, + "loss": 4.6484, + "theoretical_loss": 5.821096447332456, + "tokens_seen": 26935296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040873015873015874, + "loss": 4.6262, + "theoretical_loss": 5.818775446635469, + "tokens_seen": 27000832 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040972222222222224, + "loss": 5.0236, + "theoretical_loss": 5.816461645636238, + "tokens_seen": 27066368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004107142857142857, + "loss": 5.1234, + "theoretical_loss": 5.814155004649702, + "tokens_seen": 27131904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004117063492063492, + "loss": 4.6509, + "theoretical_loss": 5.811855484304724, + "tokens_seen": 27197440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004126984126984127, + "loss": 4.9814, + "theoretical_loss": 5.809563045540864, + "tokens_seen": 27262976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004136904761904762, + "loss": 5.0187, + "theoretical_loss": 5.8072776496051866, + "tokens_seen": 27328512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004146825396825397, + "loss": 4.9255, + "theoretical_loss": 5.804999258049106, + "tokens_seen": 27394048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004156746031746032, + "loss": 5.0265, + "theoretical_loss": 5.802727832725283, + "tokens_seen": 27459584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004166666666666667, + "loss": 4.8827, + "theoretical_loss": 5.800463335784541, + "tokens_seen": 27525120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004176587301587302, + "loss": 5.0962, + "theoretical_loss": 5.798205729672842, + "tokens_seen": 27590656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004186507936507937, + "loss": 5.0409, + "theoretical_loss": 5.795954977128286, + "tokens_seen": 27656192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041964285714285714, + "loss": 4.8332, + "theoretical_loss": 5.7937110411781525, + "tokens_seen": 27721728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 4.813, + "theoretical_loss": 5.79147388513598, + "tokens_seen": 27787264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 27295, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.60736083984375, + "objective/train/theoretical_loss": 5.789243472598683, + "objective/train/tokens_used": 48312800, + "theoretical_loss": 5.789243472598683, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042162698412698415, + "loss": 4.735, + "theoretical_loss": 5.789243472598683, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042261904761904765, + "loss": 5.0103, + "theoretical_loss": 5.787019767443696, + "tokens_seen": 27918336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004236111111111111, + "loss": 4.9487, + "theoretical_loss": 5.784802733826166, + "tokens_seen": 27983872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004246031746031746, + "loss": 5.0393, + "theoretical_loss": 5.782592336176171, + "tokens_seen": 28049408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004255952380952381, + "loss": 4.8239, + "theoretical_loss": 5.780388539195972, + "tokens_seen": 28114944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004265873015873016, + "loss": 4.8301, + "theoretical_loss": 5.778191307857307, + "tokens_seen": 28180480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042757936507936505, + "loss": 4.8818, + "theoretical_loss": 5.776000607398707, + "tokens_seen": 28246016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042857142857142855, + "loss": 5.0365, + "theoretical_loss": 5.773816403322854, + "tokens_seen": 28311552 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042956349206349205, + "loss": 4.8806, + "theoretical_loss": 5.7716386613939665, + "tokens_seen": 28377088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004305555555555556, + "loss": 4.8212, + "theoretical_loss": 5.769467347635221, + "tokens_seen": 28442624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043154761904761905, + "loss": 4.6856, + "theoretical_loss": 5.767302428326196, + "tokens_seen": 28508160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043253968253968256, + "loss": 5.1099, + "theoretical_loss": 5.765143870000358, + "tokens_seen": 28573696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043353174603174606, + "loss": 4.9942, + "theoretical_loss": 5.762991639442574, + "tokens_seen": 28639232 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043452380952380956, + "loss": 4.7269, + "theoretical_loss": 5.7608457036866465, + "tokens_seen": 28704768 + }, + { + "epoch": 0.01, + "learning_rate": 0.000435515873015873, + "loss": 4.7079, + "theoretical_loss": 5.758706030012889, + "tokens_seen": 28770304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 5.0535, + "theoretical_loss": 5.756572585945728, + "tokens_seen": 28835840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004375, + "loss": 4.9953, + "theoretical_loss": 5.754445339251326, + "tokens_seen": 28901376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004384920634920635, + "loss": 4.8239, + "theoretical_loss": 5.752324257935244, + "tokens_seen": 28966912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000439484126984127, + "loss": 4.8738, + "theoretical_loss": 5.750209310240125, + "tokens_seen": 29032448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044047619047619046, + "loss": 4.5774, + "theoretical_loss": 5.7481004646434055, + "tokens_seen": 29097984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044146825396825396, + "loss": 4.7821, + "theoretical_loss": 5.745997689855058, + "tokens_seen": 29163520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044246031746031746, + "loss": 4.9197, + "theoretical_loss": 5.743900954815356, + "tokens_seen": 29229056 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044345238095238096, + "loss": 4.9178, + "theoretical_loss": 5.741810228692663, + "tokens_seen": 29294592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004444444444444444, + "loss": 4.9811, + "theoretical_loss": 5.739725480881262, + "tokens_seen": 29360128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044543650793650797, + "loss": 4.595, + "theoretical_loss": 5.737646680999193, + "tokens_seen": 29425664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 28553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.193212032318115, + "objective/train/theoretical_loss": 5.73557379888612, + "objective/train/tokens_used": 49951200, + "theoretical_loss": 5.73557379888612, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044642857142857147, + "loss": 4.6289, + "theoretical_loss": 5.73557379888612, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044742063492063497, + "loss": 4.5175, + "theoretical_loss": 5.733506804601236, + "tokens_seen": 29556736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004484126984126984, + "loss": 4.97, + "theoretical_loss": 5.7314456684211725, + "tokens_seen": 29622272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004494047619047619, + "loss": 4.9007, + "theoretical_loss": 5.729390360837952, + "tokens_seen": 29687808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004503968253968254, + "loss": 5.0718, + "theoretical_loss": 5.7273408525569485, + "tokens_seen": 29753344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004513888888888889, + "loss": 4.8076, + "theoretical_loss": 5.725297114494884, + "tokens_seen": 29818880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 4.9151, + "theoretical_loss": 5.7232591177778405, + "tokens_seen": 29884416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045337301587301587, + "loss": 4.7611, + "theoretical_loss": 5.7212268337393, + "tokens_seen": 29949952 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045436507936507937, + "loss": 4.6533, + "theoretical_loss": 5.719200233918203, + "tokens_seen": 30015488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004553571428571429, + "loss": 4.7689, + "theoretical_loss": 5.717179290057032, + "tokens_seen": 30081024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004563492063492063, + "loss": 4.753, + "theoretical_loss": 5.715163974099917, + "tokens_seen": 30146560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004573412698412698, + "loss": 4.9783, + "theoretical_loss": 5.713154258190757, + "tokens_seen": 30212096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004583333333333333, + "loss": 4.8919, + "theoretical_loss": 5.711150114671375, + "tokens_seen": 30277632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004593253968253968, + "loss": 4.6246, + "theoretical_loss": 5.709151516079683, + "tokens_seen": 30343168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 4.8047, + "theoretical_loss": 5.707158435147875, + "tokens_seen": 30408704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046130952380952383, + "loss": 4.7619, + "theoretical_loss": 5.705170844800628, + "tokens_seen": 30474240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046230158730158733, + "loss": 5.0389, + "theoretical_loss": 5.703188718153347, + "tokens_seen": 30539776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046329365079365083, + "loss": 4.7978, + "theoretical_loss": 5.7012120285104055, + "tokens_seen": 30605312 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046428571428571433, + "loss": 4.8182, + "theoretical_loss": 5.699240749363417, + "tokens_seen": 30670848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004652777777777778, + "loss": 5.1046, + "theoretical_loss": 5.69727485438953, + "tokens_seen": 30736384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004662698412698413, + "loss": 4.9301, + "theoretical_loss": 5.695314317449732, + "tokens_seen": 30801920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004672619047619048, + "loss": 4.9513, + "theoretical_loss": 5.69335911258718, + "tokens_seen": 30867456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 4.7506, + "theoretical_loss": 5.691409214025544, + "tokens_seen": 30932992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046924603174603173, + "loss": 4.9203, + "theoretical_loss": 5.689464596167383, + "tokens_seen": 30998528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047023809523809523, + "loss": 4.6588, + "theoretical_loss": 5.687525233592513, + "tokens_seen": 31064064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 29210, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.980744361877441, + "objective/train/theoretical_loss": 5.685591101056422, + "objective/train/tokens_used": 51589600, + "theoretical_loss": 5.685591101056422, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047123015873015874, + "loss": 4.8131, + "theoretical_loss": 5.685591101056422, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047222222222222224, + "loss": 4.9729, + "theoretical_loss": 5.683662173488678, + "tokens_seen": 31195136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004732142857142857, + "loss": 4.9988, + "theoretical_loss": 5.681738425991377, + "tokens_seen": 31260672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004742063492063492, + "loss": 4.9296, + "theoretical_loss": 5.679819833837586, + "tokens_seen": 31326208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004751984126984127, + "loss": 4.8866, + "theoretical_loss": 5.677906372469826, + "tokens_seen": 31391744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 4.8619, + "theoretical_loss": 5.675998017498549, + "tokens_seen": 31457280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004771825396825397, + "loss": 4.4881, + "theoretical_loss": 5.674094744700648, + "tokens_seen": 31522816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004781746031746032, + "loss": 4.7385, + "theoretical_loss": 5.672196530017979, + "tokens_seen": 31588352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004791666666666667, + "loss": 4.9446, + "theoretical_loss": 5.670303349555893, + "tokens_seen": 31653888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004801587301587302, + "loss": 4.5805, + "theoretical_loss": 5.668415179581795, + "tokens_seen": 31719424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004811507936507937, + "loss": 4.943, + "theoretical_loss": 5.666531996523711, + "tokens_seen": 31784960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048214285714285715, + "loss": 4.9994, + "theoretical_loss": 5.664653776968867, + "tokens_seen": 31850496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048313492063492065, + "loss": 4.9432, + "theoretical_loss": 5.6627804976622995, + "tokens_seen": 31916032 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048412698412698415, + "loss": 4.4612, + "theoretical_loss": 5.660912135505461, + "tokens_seen": 31981568 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048511904761904765, + "loss": 5.0296, + "theoretical_loss": 5.659048667554853, + "tokens_seen": 32047104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004861111111111111, + "loss": 4.7342, + "theoretical_loss": 5.657190071020672, + "tokens_seen": 32112640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004871031746031746, + "loss": 5.0271, + "theoretical_loss": 5.65533632326547, + "tokens_seen": 32178176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004880952380952381, + "loss": 4.5208, + "theoretical_loss": 5.6534874018028205, + "tokens_seen": 32243712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004890873015873016, + "loss": 4.8883, + "theoretical_loss": 5.651643284296018, + "tokens_seen": 32309248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004900793650793651, + "loss": 4.5918, + "theoretical_loss": 5.6498039485567695, + "tokens_seen": 32374784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004910714285714286, + "loss": 4.8362, + "theoretical_loss": 5.6479693725439155, + "tokens_seen": 32440320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492063492063492, + "loss": 4.4816, + "theoretical_loss": 5.646139534362161, + "tokens_seen": 32505856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004930555555555556, + "loss": 4.7304, + "theoretical_loss": 5.644314412260815, + "tokens_seen": 32571392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004940476190476191, + "loss": 4.7639, + "theoretical_loss": 5.642493984632544, + "tokens_seen": 32636928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004950396825396826, + "loss": 4.486, + "theoretical_loss": 5.640678230012151, + "tokens_seen": 32702464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 30192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.205135822296143, + "objective/train/theoretical_loss": 5.638867127075349, + "objective/train/tokens_used": 53228000, + "theoretical_loss": 5.638867127075349, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496031746031746, + "loss": 4.6383, + "theoretical_loss": 5.638867127075349, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004970238095238095, + "loss": 4.7665, + "theoretical_loss": 5.63706065463756, + "tokens_seen": 32833536 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498015873015873, + "loss": 4.6651, + "theoretical_loss": 5.635258791652723, + "tokens_seen": 32899072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990079365079365, + "loss": 4.6032, + "theoretical_loss": 5.633461517212115, + "tokens_seen": 32964608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 4.8611, + "theoretical_loss": 5.631668810543181, + "tokens_seen": 33030144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999899699097292, + "loss": 4.4403, + "theoretical_loss": 5.629880651008384, + "tokens_seen": 33095680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999799398194584, + "loss": 4.6342, + "theoretical_loss": 5.628097018104057, + "tokens_seen": 33161216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999699097291876, + "loss": 4.789, + "theoretical_loss": 5.626317891459275, + "tokens_seen": 33226752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999598796389167, + "loss": 4.4669, + "theoretical_loss": 5.6245432508347335, + "tokens_seen": 33292288 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499949849548646, + "loss": 4.5265, + "theoretical_loss": 5.622773076121648, + "tokens_seen": 33357824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999398194583751, + "loss": 4.8668, + "theoretical_loss": 5.621007347340646, + "tokens_seen": 33423360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999297893681044, + "loss": 4.663, + "theoretical_loss": 5.619246044640693, + "tokens_seen": 33488896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999197592778335, + "loss": 4.51, + "theoretical_loss": 5.617489148298015, + "tokens_seen": 33554432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999097291875627, + "loss": 4.4827, + "theoretical_loss": 5.615736638715033, + "tokens_seen": 33619968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998996990972919, + "loss": 4.6552, + "theoretical_loss": 5.613988496419315, + "tokens_seen": 33685504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998896690070211, + "loss": 4.5444, + "theoretical_loss": 5.6122447020625374, + "tokens_seen": 33751040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998796389167503, + "loss": 4.7462, + "theoretical_loss": 5.610505236419448, + "tokens_seen": 33816576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998696088264795, + "loss": 4.6343, + "theoretical_loss": 5.608770080386854, + "tokens_seen": 33882112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998595787362087, + "loss": 4.3194, + "theoretical_loss": 5.6070392149826045, + "tokens_seen": 33947648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998495486459378, + "loss": 4.7376, + "theoretical_loss": 5.605312621344602, + "tokens_seen": 34013184 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499839518555667, + "loss": 4.6319, + "theoretical_loss": 5.603590280729803, + "tokens_seen": 34078720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998294884653962, + "loss": 4.3936, + "theoretical_loss": 5.601872174513245, + "tokens_seen": 34144256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998194583751254, + "loss": 4.8045, + "theoretical_loss": 5.60015828418708, + "tokens_seen": 34209792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998094282848546, + "loss": 4.8346, + "theoretical_loss": 5.59844859135961, + "tokens_seen": 34275328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997993981945837, + "loss": 4.4444, + "theoretical_loss": 5.596743077754337, + "tokens_seen": 34340864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 30758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.619051933288574, + "objective/train/theoretical_loss": 5.5950417252090325, + "objective/train/tokens_used": 54866400, + "theoretical_loss": 5.5950417252090325, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997893681043129, + "loss": 4.5684, + "theoretical_loss": 5.5950417252090325, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997793380140421, + "loss": 4.7403, + "theoretical_loss": 5.593344515674795, + "tokens_seen": 34471936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997693079237714, + "loss": 4.7682, + "theoretical_loss": 5.591651431215139, + "tokens_seen": 34537472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997592778335005, + "loss": 4.9194, + "theoretical_loss": 5.589962454005072, + "tokens_seen": 34603008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997492477432298, + "loss": 4.7926, + "theoretical_loss": 5.588277566330207, + "tokens_seen": 34668544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997392176529588, + "loss": 4.5919, + "theoretical_loss": 5.586596750585855, + "tokens_seen": 34734080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997291875626881, + "loss": 4.8471, + "theoretical_loss": 5.584919989276146, + "tokens_seen": 34799616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997191574724173, + "loss": 4.8823, + "theoretical_loss": 5.583247265013155, + "tokens_seen": 34865152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997091273821465, + "loss": 4.7952, + "theoretical_loss": 5.581578560516027, + "tokens_seen": 34930688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996990972918757, + "loss": 4.8327, + "theoretical_loss": 5.57991385861013, + "tokens_seen": 34996224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996890672016048, + "loss": 4.3745, + "theoretical_loss": 5.578253142226191, + "tokens_seen": 35061760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499679037111334, + "loss": 4.6921, + "theoretical_loss": 5.576596394399467, + "tokens_seen": 35127296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996690070210632, + "loss": 4.6698, + "theoretical_loss": 5.574943598268902, + "tokens_seen": 35192832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996589769307924, + "loss": 4.9374, + "theoretical_loss": 5.573294737076306, + "tokens_seen": 35258368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996489468405216, + "loss": 4.5814, + "theoretical_loss": 5.571649794165537, + "tokens_seen": 35323904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996389167502507, + "loss": 4.6064, + "theoretical_loss": 5.570008752981688, + "tokens_seen": 35389440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996288866599799, + "loss": 4.7137, + "theoretical_loss": 5.5683715970702945, + "tokens_seen": 35454976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996188565697091, + "loss": 4.7919, + "theoretical_loss": 5.566738310076531, + "tokens_seen": 35520512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996088264794383, + "loss": 4.5989, + "theoretical_loss": 5.5651088757444285, + "tokens_seen": 35586048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995987963891675, + "loss": 4.4273, + "theoretical_loss": 5.5634832779161, + "tokens_seen": 35651584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995887662988968, + "loss": 4.6147, + "theoretical_loss": 5.561861500530965, + "tokens_seen": 35717120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995787362086258, + "loss": 4.5803, + "theoretical_loss": 5.560243527624989, + "tokens_seen": 35782656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995687061183551, + "loss": 4.762, + "theoretical_loss": 5.558629343329924, + "tokens_seen": 35848192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995586760280842, + "loss": 4.7117, + "theoretical_loss": 5.557018931872566, + "tokens_seen": 35913728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995486459378135, + "loss": 4.7915, + "theoretical_loss": 5.555412277574007, + "tokens_seen": 35979264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 31707, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.547565460205078, + "objective/train/theoretical_loss": 5.553809364848902, + "objective/train/tokens_used": 56504800, + "theoretical_loss": 5.553809364848902, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995386158475427, + "loss": 4.6425, + "theoretical_loss": 5.553809364848902, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995285857572718, + "loss": 4.5963, + "theoretical_loss": 5.552210178204751, + "tokens_seen": 36110336 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499518555667001, + "loss": 4.6793, + "theoretical_loss": 5.550614702241162, + "tokens_seen": 36175872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995085255767302, + "loss": 4.7561, + "theoretical_loss": 5.549022921649151, + "tokens_seen": 36241408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994984954864594, + "loss": 4.6607, + "theoretical_loss": 5.54743482121043, + "tokens_seen": 36306944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994884653961886, + "loss": 4.6252, + "theoretical_loss": 5.545850385796712, + "tokens_seen": 36372480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994784353059178, + "loss": 4.4331, + "theoretical_loss": 5.544269600369009, + "tokens_seen": 36438016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994684052156469, + "loss": 4.643, + "theoretical_loss": 5.5426924499769505, + "tokens_seen": 36503552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994583751253761, + "loss": 4.7548, + "theoretical_loss": 5.541118919758107, + "tokens_seen": 36569088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994483450351053, + "loss": 4.7337, + "theoretical_loss": 5.539548994937308, + "tokens_seen": 36634624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994383149448345, + "loss": 4.3641, + "theoretical_loss": 5.537982660825981, + "tokens_seen": 36700160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994282848545637, + "loss": 4.5599, + "theoretical_loss": 5.536419902821491, + "tokens_seen": 36765696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994182547642928, + "loss": 4.6875, + "theoretical_loss": 5.534860706406477, + "tokens_seen": 36831232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994082246740221, + "loss": 4.7352, + "theoretical_loss": 5.533305057148215, + "tokens_seen": 36896768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993981945837512, + "loss": 4.6149, + "theoretical_loss": 5.531752940697968, + "tokens_seen": 36962304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993881644934805, + "loss": 4.5067, + "theoretical_loss": 5.5302043427903484, + "tokens_seen": 37027840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993781344032096, + "loss": 4.9474, + "theoretical_loss": 5.528659249242693, + "tokens_seen": 37093376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993681043129389, + "loss": 4.4017, + "theoretical_loss": 5.527117645954435, + "tokens_seen": 37158912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499358074222668, + "loss": 4.5323, + "theoretical_loss": 5.525579518906484, + "tokens_seen": 37224448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993480441323972, + "loss": 4.4802, + "theoretical_loss": 5.524044854160613, + "tokens_seen": 37289984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993380140421264, + "loss": 4.4531, + "theoretical_loss": 5.522513637858855, + "tokens_seen": 37355520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993279839518556, + "loss": 4.6216, + "theoretical_loss": 5.520985856222893, + "tokens_seen": 37421056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993179538615848, + "loss": 4.5407, + "theoretical_loss": 5.519461495553473, + "tokens_seen": 37486592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993079237713139, + "loss": 4.4097, + "theoretical_loss": 5.5179405422298045, + "tokens_seen": 37552128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992978936810431, + "loss": 4.406, + "theoretical_loss": 5.516422982708983, + "tokens_seen": 37617664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 32203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.049692630767822, + "objective/train/theoretical_loss": 5.514908803525402, + "objective/train/tokens_used": 58143200, + "theoretical_loss": 5.514908803525402, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992878635907723, + "loss": 4.3638, + "theoretical_loss": 5.514908803525402, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992778335005015, + "loss": 4.6329, + "theoretical_loss": 5.513397991290185, + "tokens_seen": 37748736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992678034102307, + "loss": 4.6538, + "theoretical_loss": 5.511890532690611, + "tokens_seen": 37814272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992577733199598, + "loss": 4.7513, + "theoretical_loss": 5.510386414489556, + "tokens_seen": 37879808 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499247743229689, + "loss": 4.6591, + "theoretical_loss": 5.5088856235249235, + "tokens_seen": 37945344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992377131394183, + "loss": 4.4618, + "theoretical_loss": 5.507388146709105, + "tokens_seen": 38010880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992276830491475, + "loss": 4.4231, + "theoretical_loss": 5.505893971028418, + "tokens_seen": 38076416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992176529588767, + "loss": 4.2713, + "theoretical_loss": 5.504403083542568, + "tokens_seen": 38141952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992076228686059, + "loss": 4.4092, + "theoretical_loss": 5.502915471384111, + "tokens_seen": 38207488 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499197592778335, + "loss": 4.7235, + "theoretical_loss": 5.501431121757914, + "tokens_seen": 38273024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991875626880642, + "loss": 4.3762, + "theoretical_loss": 5.49995002194063, + "tokens_seen": 38338560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991775325977934, + "loss": 4.6923, + "theoretical_loss": 5.498472159280171, + "tokens_seen": 38404096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991675025075226, + "loss": 4.8392, + "theoretical_loss": 5.4969975211951905, + "tokens_seen": 38469632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991574724172518, + "loss": 4.5345, + "theoretical_loss": 5.495526095174563, + "tokens_seen": 38535168 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499147442326981, + "loss": 4.6407, + "theoretical_loss": 5.494057868776883, + "tokens_seen": 38600704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991374122367101, + "loss": 4.5119, + "theoretical_loss": 5.492592829629946, + "tokens_seen": 38666240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991273821464393, + "loss": 4.6456, + "theoretical_loss": 5.491130965430262, + "tokens_seen": 38731776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991173520561685, + "loss": 4.2981, + "theoretical_loss": 5.489672263942543, + "tokens_seen": 38797312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991073219658977, + "loss": 4.7488, + "theoretical_loss": 5.488216712999222, + "tokens_seen": 38862848 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499097291875627, + "loss": 4.5654, + "theoretical_loss": 5.486764300499962, + "tokens_seen": 38928384 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499087261785356, + "loss": 4.3462, + "theoretical_loss": 5.4853150144111655, + "tokens_seen": 38993920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990772316950853, + "loss": 4.6389, + "theoretical_loss": 5.4838688427655065, + "tokens_seen": 39059456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990672016048144, + "loss": 4.5166, + "theoretical_loss": 5.482425773661445, + "tokens_seen": 39124992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990571715145437, + "loss": 4.5858, + "theoretical_loss": 5.480985795262759, + "tokens_seen": 39190528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990471414242729, + "loss": 4.6098, + "theoretical_loss": 5.479548895798082, + "tokens_seen": 39256064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 33134, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.761120319366455, + "objective/train/theoretical_loss": 5.478115063560433, + "objective/train/tokens_used": 59781600, + "theoretical_loss": 5.478115063560433, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499037111334002, + "loss": 4.4957, + "theoretical_loss": 5.478115063560433, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990270812437312, + "loss": 4.7046, + "theoretical_loss": 5.476684286906764, + "tokens_seen": 39387136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990170511534604, + "loss": 4.2948, + "theoretical_loss": 5.475256554257497, + "tokens_seen": 39452672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990070210631896, + "loss": 4.5969, + "theoretical_loss": 5.473831854096085, + "tokens_seen": 39518208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989969909729188, + "loss": 4.5511, + "theoretical_loss": 5.472410174968553, + "tokens_seen": 39583744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498986960882648, + "loss": 4.4106, + "theoretical_loss": 5.470991505483065, + "tokens_seen": 39649280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989769307923771, + "loss": 4.623, + "theoretical_loss": 5.469575834309477, + "tokens_seen": 39714816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989669007021063, + "loss": 4.5077, + "theoretical_loss": 5.468163150178907, + "tokens_seen": 39780352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989568706118355, + "loss": 4.475, + "theoretical_loss": 5.466753441883302, + "tokens_seen": 39845888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989468405215647, + "loss": 4.6451, + "theoretical_loss": 5.465346698275008, + "tokens_seen": 39911424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989368104312939, + "loss": 4.3366, + "theoretical_loss": 5.46394290826635, + "tokens_seen": 39976960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498926780341023, + "loss": 4.4816, + "theoretical_loss": 5.462542060829209, + "tokens_seen": 40042496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989167502507523, + "loss": 4.5005, + "theoretical_loss": 5.461144144994606, + "tokens_seen": 40108032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989067201604814, + "loss": 4.5075, + "theoretical_loss": 5.459749149852287, + "tokens_seen": 40173568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988966900702107, + "loss": 4.555, + "theoretical_loss": 5.4583570645503166, + "tokens_seen": 40239104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988866599799398, + "loss": 4.411, + "theoretical_loss": 5.45696787829467, + "tokens_seen": 40304640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988766298896691, + "loss": 4.479, + "theoretical_loss": 5.455581580348831, + "tokens_seen": 40370176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988665997993982, + "loss": 4.424, + "theoretical_loss": 5.454198160033391, + "tokens_seen": 40435712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988565697091274, + "loss": 4.6216, + "theoretical_loss": 5.452817606725657, + "tokens_seen": 40501248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988465396188566, + "loss": 4.5915, + "theoretical_loss": 5.451439909859255, + "tokens_seen": 40566784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988365095285858, + "loss": 4.4225, + "theoretical_loss": 5.450065058923747, + "tokens_seen": 40632320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498826479438315, + "loss": 4.4659, + "theoretical_loss": 5.448693043464235, + "tokens_seen": 40697856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988164493480441, + "loss": 4.7231, + "theoretical_loss": 5.447323853080991, + "tokens_seen": 40763392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988064192577733, + "loss": 4.561, + "theoretical_loss": 5.445957477429067, + "tokens_seen": 40828928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987963891675025, + "loss": 4.4559, + "theoretical_loss": 5.444593906217927, + "tokens_seen": 40894464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 33820, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.709114074707031, + "objective/train/theoretical_loss": 5.443233129211069, + "objective/train/tokens_used": 61420000, + "theoretical_loss": 5.443233129211069, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987863590772317, + "loss": 4.5353, + "theoretical_loss": 5.443233129211069, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987763289869609, + "loss": 4.5888, + "theoretical_loss": 5.441875136225658, + "tokens_seen": 41025536 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049876629889669, + "loss": 4.6609, + "theoretical_loss": 5.440519917132159, + "tokens_seen": 41091072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987562688064192, + "loss": 4.6782, + "theoretical_loss": 5.439167461853971, + "tokens_seen": 41156608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987462387161484, + "loss": 4.3218, + "theoretical_loss": 5.437817760367073, + "tokens_seen": 41222144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987362086258777, + "loss": 4.6102, + "theoretical_loss": 5.436470802699661, + "tokens_seen": 41287680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987261785356068, + "loss": 4.4044, + "theoretical_loss": 5.43512657893179, + "tokens_seen": 41353216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987161484453361, + "loss": 4.5902, + "theoretical_loss": 5.433785079195035, + "tokens_seen": 41418752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987061183550651, + "loss": 4.4395, + "theoretical_loss": 5.432446293672129, + "tokens_seen": 41484288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986960882647944, + "loss": 4.6268, + "theoretical_loss": 5.43111021259663, + "tokens_seen": 41549824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986860581745236, + "loss": 4.7247, + "theoretical_loss": 5.429776826252565, + "tokens_seen": 41615360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986760280842528, + "loss": 4.7844, + "theoretical_loss": 5.428446124974103, + "tokens_seen": 41680896 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498665997993982, + "loss": 4.748, + "theoretical_loss": 5.427118099145207, + "tokens_seen": 41746432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986559679037111, + "loss": 4.4521, + "theoretical_loss": 5.425792739199306, + "tokens_seen": 41811968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986459378134403, + "loss": 4.5162, + "theoretical_loss": 5.424470035618963, + "tokens_seen": 41877504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986359077231695, + "loss": 4.5787, + "theoretical_loss": 5.423149978935547, + "tokens_seen": 41943040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986258776328987, + "loss": 4.4447, + "theoretical_loss": 5.421832559728902, + "tokens_seen": 42008576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986158475426279, + "loss": 4.5436, + "theoretical_loss": 5.420517768627027, + "tokens_seen": 42074112 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498605817452357, + "loss": 4.5508, + "theoretical_loss": 5.419205596305758, + "tokens_seen": 42139648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985957873620862, + "loss": 4.3851, + "theoretical_loss": 5.417896033488446, + "tokens_seen": 42205184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985857572718154, + "loss": 4.4232, + "theoretical_loss": 5.416589070945641, + "tokens_seen": 42270720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985757271815446, + "loss": 4.4672, + "theoretical_loss": 5.415284699494782, + "tokens_seen": 42336256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985656970912738, + "loss": 4.3494, + "theoretical_loss": 5.413982909999887, + "tokens_seen": 42401792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985556670010031, + "loss": 4.246, + "theoretical_loss": 5.412683693371241, + "tokens_seen": 42467328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985456369107321, + "loss": 4.5265, + "theoretical_loss": 5.411387040565092, + "tokens_seen": 42532864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 34938, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.999885082244873, + "objective/train/theoretical_loss": 5.410092942583357, + "objective/train/tokens_used": 63058400, + "theoretical_loss": 5.410092942583357, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985356068204614, + "loss": 4.5919, + "theoretical_loss": 5.410092942583357, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985255767301905, + "loss": 4.603, + "theoretical_loss": 5.408801390473304, + "tokens_seen": 42663936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985155466399198, + "loss": 4.546, + "theoretical_loss": 5.407512375327272, + "tokens_seen": 42729472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498505516549649, + "loss": 4.5326, + "theoretical_loss": 5.406225888282363, + "tokens_seen": 42795008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984954864593782, + "loss": 4.3889, + "theoretical_loss": 5.404941920520156, + "tokens_seen": 42860544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984854563691073, + "loss": 4.1785, + "theoretical_loss": 5.403660463266413, + "tokens_seen": 42926080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984754262788365, + "loss": 4.5382, + "theoretical_loss": 5.402381507790792, + "tokens_seen": 42991616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984653961885657, + "loss": 4.5693, + "theoretical_loss": 5.401105045406557, + "tokens_seen": 43057152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984553660982949, + "loss": 4.4403, + "theoretical_loss": 5.399831067470302, + "tokens_seen": 43122688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984453360080241, + "loss": 4.7214, + "theoretical_loss": 5.398559565381662, + "tokens_seen": 43188224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984353059177532, + "loss": 4.4442, + "theoretical_loss": 5.397290530583037, + "tokens_seen": 43253760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984252758274825, + "loss": 4.309, + "theoretical_loss": 5.396023954559315, + "tokens_seen": 43319296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984152457372116, + "loss": 4.3474, + "theoretical_loss": 5.394759828837596, + "tokens_seen": 43384832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984052156469409, + "loss": 4.5364, + "theoretical_loss": 5.393498144986921, + "tokens_seen": 43450368 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049839518555667, + "loss": 4.647, + "theoretical_loss": 5.392238894618, + "tokens_seen": 43515904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983851554663993, + "loss": 4.4688, + "theoretical_loss": 5.390982069382944, + "tokens_seen": 43581440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983751253761284, + "loss": 4.555, + "theoretical_loss": 5.389727660974998, + "tokens_seen": 43646976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983650952858576, + "loss": 4.2673, + "theoretical_loss": 5.38847566112828, + "tokens_seen": 43712512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983550651955868, + "loss": 4.223, + "theoretical_loss": 5.387226061617516, + "tokens_seen": 43778048 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498345035105316, + "loss": 4.5944, + "theoretical_loss": 5.38597885425778, + "tokens_seen": 43843584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983350050150452, + "loss": 4.2395, + "theoretical_loss": 5.384734030904238, + "tokens_seen": 43909120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983249749247743, + "loss": 4.2047, + "theoretical_loss": 5.383491583451892, + "tokens_seen": 43974656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983149448345035, + "loss": 4.6784, + "theoretical_loss": 5.382251503835324, + "tokens_seen": 44040192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983049147442327, + "loss": 4.5094, + "theoretical_loss": 5.381013784028445, + "tokens_seen": 44105728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982948846539619, + "loss": 4.2719, + "theoretical_loss": 5.379778416044247, + "tokens_seen": 44171264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 35534, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.2067718505859375, + "objective/train/theoretical_loss": 5.378545391934553, + "objective/train/tokens_used": 64696800, + "theoretical_loss": 5.378545391934553, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982848545636911, + "loss": 4.3011, + "theoretical_loss": 5.378545391934553, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982748244734202, + "loss": 4.3593, + "theoretical_loss": 5.377314703789773, + "tokens_seen": 44302336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982647943831494, + "loss": 4.5546, + "theoretical_loss": 5.376086343738656, + "tokens_seen": 44367872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982547642928786, + "loss": 4.2113, + "theoretical_loss": 5.3748603039480525, + "tokens_seen": 44433408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982447342026079, + "loss": 4.6548, + "theoretical_loss": 5.373636576622671, + "tokens_seen": 44498944 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498234704112337, + "loss": 4.6067, + "theoretical_loss": 5.372415154004842, + "tokens_seen": 44564480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982246740220663, + "loss": 4.5434, + "theoretical_loss": 5.371196028374278, + "tokens_seen": 44630016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982146439317953, + "loss": 4.219, + "theoretical_loss": 5.369979192047846, + "tokens_seen": 44695552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982046138415246, + "loss": 4.4616, + "theoretical_loss": 5.368764637379322, + "tokens_seen": 44761088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981945837512538, + "loss": 4.3927, + "theoretical_loss": 5.367552356759175, + "tokens_seen": 44826624 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498184553660983, + "loss": 4.3436, + "theoretical_loss": 5.366342342614322, + "tokens_seen": 44892160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981745235707122, + "loss": 4.3641, + "theoretical_loss": 5.365134587407918, + "tokens_seen": 44957696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981644934804413, + "loss": 4.3878, + "theoretical_loss": 5.363929083639112, + "tokens_seen": 45023232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981544633901705, + "loss": 4.252, + "theoretical_loss": 5.362725823842838, + "tokens_seen": 45088768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981444332998997, + "loss": 4.1657, + "theoretical_loss": 5.361524800589581, + "tokens_seen": 45154304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981344032096289, + "loss": 4.0417, + "theoretical_loss": 5.360326006485165, + "tokens_seen": 45219840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981243731193581, + "loss": 4.6023, + "theoretical_loss": 5.359129434170529, + "tokens_seen": 45285376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981143430290873, + "loss": 4.3602, + "theoretical_loss": 5.357935076321514, + "tokens_seen": 45350912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981043129388164, + "loss": 4.4446, + "theoretical_loss": 5.35674292564864, + "tokens_seen": 45416448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980942828485456, + "loss": 4.4122, + "theoretical_loss": 5.3555529748969, + "tokens_seen": 45481984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980842527582748, + "loss": 4.465, + "theoretical_loss": 5.354365216845546, + "tokens_seen": 45547520 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498074222668004, + "loss": 4.1501, + "theoretical_loss": 5.353179644307868, + "tokens_seen": 45613056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980641925777333, + "loss": 4.6183, + "theoretical_loss": 5.351996250131005, + "tokens_seen": 45678592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980541624874623, + "loss": 4.4826, + "theoretical_loss": 5.350815027195717, + "tokens_seen": 45744128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980441323971916, + "loss": 4.0969, + "theoretical_loss": 5.349635968416189, + "tokens_seen": 45809664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 36791, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.983624219894409, + "objective/train/theoretical_loss": 5.34845906673983, + "objective/train/tokens_used": 66335200, + "theoretical_loss": 5.34845906673983, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980341023069207, + "loss": 4.1114, + "theoretical_loss": 5.34845906673983, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049802407221665, + "loss": 4.3748, + "theoretical_loss": 5.347284315147059, + "tokens_seen": 45940736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980140421263792, + "loss": 4.3086, + "theoretical_loss": 5.346111706651114, + "tokens_seen": 46006272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980040120361084, + "loss": 4.5521, + "theoretical_loss": 5.344941234297847, + "tokens_seen": 46071808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979939819458375, + "loss": 4.4492, + "theoretical_loss": 5.343772891165527, + "tokens_seen": 46137344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979839518555667, + "loss": 4.4528, + "theoretical_loss": 5.34260667036464, + "tokens_seen": 46202880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979739217652959, + "loss": 4.6275, + "theoretical_loss": 5.3414425650377, + "tokens_seen": 46268416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979638916750251, + "loss": 4.5955, + "theoretical_loss": 5.3402805683590495, + "tokens_seen": 46333952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979538615847543, + "loss": 4.5274, + "theoretical_loss": 5.339120673534668, + "tokens_seen": 46399488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979438314944834, + "loss": 4.1984, + "theoretical_loss": 5.337962873801984, + "tokens_seen": 46465024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979338014042126, + "loss": 4.4472, + "theoretical_loss": 5.336807162429681, + "tokens_seen": 46530560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979237713139418, + "loss": 4.2688, + "theoretical_loss": 5.335653532717512, + "tokens_seen": 46596096 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497913741223671, + "loss": 4.5101, + "theoretical_loss": 5.33450197799611, + "tokens_seen": 46661632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979037111334002, + "loss": 4.4218, + "theoretical_loss": 5.33335249162681, + "tokens_seen": 46727168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978936810431293, + "loss": 4.365, + "theoretical_loss": 5.332205067001454, + "tokens_seen": 46792704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978836509528586, + "loss": 4.4012, + "theoretical_loss": 5.331059697542216, + "tokens_seen": 46858240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978736208625877, + "loss": 4.178, + "theoretical_loss": 5.329916376701417, + "tokens_seen": 46923776 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497863590772317, + "loss": 4.404, + "theoretical_loss": 5.328775097961346, + "tokens_seen": 46989312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978535606820461, + "loss": 4.3063, + "theoretical_loss": 5.327635854834084, + "tokens_seen": 47054848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978435305917754, + "loss": 4.3593, + "theoretical_loss": 5.326498640861321, + "tokens_seen": 47120384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978335005015045, + "loss": 4.458, + "theoretical_loss": 5.325363449614185, + "tokens_seen": 47185920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978234704112337, + "loss": 4.4505, + "theoretical_loss": 5.324230274693065, + "tokens_seen": 47251456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978134403209629, + "loss": 4.2784, + "theoretical_loss": 5.323099109727434, + "tokens_seen": 47316992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978034102306921, + "loss": 4.3645, + "theoretical_loss": 5.321969948375687, + "tokens_seen": 47382528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977933801404213, + "loss": 4.3349, + "theoretical_loss": 5.320842784324954, + "tokens_seen": 47448064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 38014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.285874366760254, + "objective/train/theoretical_loss": 5.319717611290951, + "objective/train/tokens_used": 67973600, + "theoretical_loss": 5.319717611290951, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977833500501504, + "loss": 4.2487, + "theoretical_loss": 5.319717611290951, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977733199598796, + "loss": 4.227, + "theoretical_loss": 5.318594423017791, + "tokens_seen": 47579136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977632898696088, + "loss": 4.5068, + "theoretical_loss": 5.317473213277827, + "tokens_seen": 47644672 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497753259779338, + "loss": 4.4264, + "theoretical_loss": 5.316353975871486, + "tokens_seen": 47710208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977432296890672, + "loss": 4.4848, + "theoretical_loss": 5.315236704627106, + "tokens_seen": 47775744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977331995987965, + "loss": 4.4288, + "theoretical_loss": 5.314121393400763, + "tokens_seen": 47841280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977231695085255, + "loss": 4.0077, + "theoretical_loss": 5.313008036076115, + "tokens_seen": 47906816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977131394182548, + "loss": 4.3322, + "theoretical_loss": 5.311896626564246, + "tokens_seen": 47972352 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497703109327984, + "loss": 4.1105, + "theoretical_loss": 5.3107871588034925, + "tokens_seen": 48037888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976930792377132, + "loss": 4.3437, + "theoretical_loss": 5.309679626759296, + "tokens_seen": 48103424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976830491474424, + "loss": 4.292, + "theoretical_loss": 5.308574024424041, + "tokens_seen": 48168960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976730190571715, + "loss": 4.3868, + "theoretical_loss": 5.3074703458168955, + "tokens_seen": 48234496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976629889669007, + "loss": 4.436, + "theoretical_loss": 5.3063685849836615, + "tokens_seen": 48300032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976529588766299, + "loss": 4.368, + "theoretical_loss": 5.305268735996615, + "tokens_seen": 48365568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976429287863591, + "loss": 4.2524, + "theoretical_loss": 5.304170792954354, + "tokens_seen": 48431104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976328986960883, + "loss": 4.5041, + "theoretical_loss": 5.303074749981648, + "tokens_seen": 48496640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976228686058175, + "loss": 4.412, + "theoretical_loss": 5.301980601229285, + "tokens_seen": 48562176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976128385155466, + "loss": 4.3484, + "theoretical_loss": 5.300888340873924, + "tokens_seen": 48627712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976028084252758, + "loss": 4.4722, + "theoretical_loss": 5.299797963117941, + "tokens_seen": 48693248 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497592778335005, + "loss": 4.1945, + "theoretical_loss": 5.298709462189283, + "tokens_seen": 48758784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975827482447342, + "loss": 4.2003, + "theoretical_loss": 5.297622832341327, + "tokens_seen": 48824320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975727181544635, + "loss": 4.0957, + "theoretical_loss": 5.2965380678527225, + "tokens_seen": 48889856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975626880641925, + "loss": 4.2418, + "theoretical_loss": 5.295455163027258, + "tokens_seen": 48955392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975526579739218, + "loss": 4.3033, + "theoretical_loss": 5.29437411219371, + "tokens_seen": 49020928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975426278836509, + "loss": 4.2827, + "theoretical_loss": 5.293294909705702, + "tokens_seen": 49086464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 38689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.5455546379089355, + "objective/train/theoretical_loss": 5.292217549941562, + "objective/train/tokens_used": 69612000, + "theoretical_loss": 5.292217549941562, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975325977933802, + "loss": 4.2752, + "theoretical_loss": 5.292217549941562, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975225677031094, + "loss": 4.1889, + "theoretical_loss": 5.291142027304185, + "tokens_seen": 49217536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975125376128386, + "loss": 3.9788, + "theoretical_loss": 5.29006833622089, + "tokens_seen": 49283072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975025075225677, + "loss": 4.4666, + "theoretical_loss": 5.28899647114328, + "tokens_seen": 49348608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974924774322969, + "loss": 4.3282, + "theoretical_loss": 5.287926426547106, + "tokens_seen": 49414144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974824473420261, + "loss": 4.3775, + "theoretical_loss": 5.2868581969321315, + "tokens_seen": 49479680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974724172517553, + "loss": 4.3022, + "theoretical_loss": 5.285791776821995, + "tokens_seen": 49545216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974623871614845, + "loss": 4.559, + "theoretical_loss": 5.2847271607640725, + "tokens_seen": 49610752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974523570712136, + "loss": 4.5333, + "theoretical_loss": 5.283664343329346, + "tokens_seen": 49676288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974423269809428, + "loss": 4.2267, + "theoretical_loss": 5.282603319112273, + "tokens_seen": 49741824 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497432296890672, + "loss": 4.3567, + "theoretical_loss": 5.281544082730649, + "tokens_seen": 49807360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974222668004012, + "loss": 4.2938, + "theoretical_loss": 5.280486628825479, + "tokens_seen": 49872896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974122367101304, + "loss": 4.3754, + "theoretical_loss": 5.279430952060843, + "tokens_seen": 49938432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974022066198595, + "loss": 4.3361, + "theoretical_loss": 5.278377047123777, + "tokens_seen": 50003968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973921765295888, + "loss": 4.2186, + "theoretical_loss": 5.277324908724132, + "tokens_seen": 50069504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973821464393179, + "loss": 4.2978, + "theoretical_loss": 5.276274531594453, + "tokens_seen": 50135040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973721163490472, + "loss": 4.3052, + "theoretical_loss": 5.27522591048985, + "tokens_seen": 50200576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973620862587763, + "loss": 4.352, + "theoretical_loss": 5.274179040187876, + "tokens_seen": 50266112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973520561685056, + "loss": 4.0427, + "theoretical_loss": 5.273133915488394, + "tokens_seen": 50331648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973420260782347, + "loss": 4.3078, + "theoretical_loss": 5.272090531213462, + "tokens_seen": 50397184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973319959879639, + "loss": 4.0645, + "theoretical_loss": 5.271048882207202, + "tokens_seen": 50462720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973219658976931, + "loss": 4.6409, + "theoretical_loss": 5.27000896333568, + "tokens_seen": 50528256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973119358074223, + "loss": 4.0496, + "theoretical_loss": 5.268970769486789, + "tokens_seen": 50593792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973019057171515, + "loss": 4.4574, + "theoretical_loss": 5.267934295570118, + "tokens_seen": 50659328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972918756268806, + "loss": 4.2343, + "theoretical_loss": 5.266899536516842, + "tokens_seen": 50724864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 39885, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.198443412780762, + "objective/train/theoretical_loss": 5.265866487279597, + "objective/train/tokens_used": 71250400, + "theoretical_loss": 5.265866487279597, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972818455366098, + "loss": 4.2608, + "theoretical_loss": 5.265866487279597, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497271815446339, + "loss": 4.2763, + "theoretical_loss": 5.264835142832364, + "tokens_seen": 50855936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972617853560682, + "loss": 4.1906, + "theoretical_loss": 5.263805498170351, + "tokens_seen": 50921472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972517552657974, + "loss": 4.1761, + "theoretical_loss": 5.26277754830987, + "tokens_seen": 50987008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972417251755266, + "loss": 4.2526, + "theoretical_loss": 5.261751288288236, + "tokens_seen": 51052544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972316950852557, + "loss": 4.275, + "theoretical_loss": 5.260726713163637, + "tokens_seen": 51118080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972216649949849, + "loss": 4.3822, + "theoretical_loss": 5.259703818015023, + "tokens_seen": 51183616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972116349047142, + "loss": 4.235, + "theoretical_loss": 5.258682597941998, + "tokens_seen": 51249152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972016048144433, + "loss": 4.1933, + "theoretical_loss": 5.257663048064704, + "tokens_seen": 51314688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971915747241726, + "loss": 4.0819, + "theoretical_loss": 5.256645163523704, + "tokens_seen": 51380224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971815446339017, + "loss": 4.2302, + "theoretical_loss": 5.255628939479877, + "tokens_seen": 51445760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971715145436309, + "loss": 4.1741, + "theoretical_loss": 5.254614371114306, + "tokens_seen": 51511296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971614844533601, + "loss": 4.0105, + "theoretical_loss": 5.253601453628165, + "tokens_seen": 51576832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971514543630893, + "loss": 4.1921, + "theoretical_loss": 5.2525901822426135, + "tokens_seen": 51642368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971414242728185, + "loss": 4.1033, + "theoretical_loss": 5.251580552198687, + "tokens_seen": 51707904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313941825477, + "loss": 4.5223, + "theoretical_loss": 5.250572558757186, + "tokens_seen": 51773440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971213640922768, + "loss": 4.2685, + "theoretical_loss": 5.2495661971985745, + "tokens_seen": 51838976 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497111334002006, + "loss": 4.4627, + "theoretical_loss": 5.248561462822871, + "tokens_seen": 51904512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971013039117352, + "loss": 4.2511, + "theoretical_loss": 5.247558350949539, + "tokens_seen": 51970048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970912738214644, + "loss": 4.0554, + "theoretical_loss": 5.246556856917392, + "tokens_seen": 52035584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970812437311936, + "loss": 4.1646, + "theoretical_loss": 5.24555697608448, + "tokens_seen": 52101120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970712136409227, + "loss": 4.2295, + "theoretical_loss": 5.244558703827986, + "tokens_seen": 52166656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970611835506519, + "loss": 4.1183, + "theoretical_loss": 5.243562035544132, + "tokens_seen": 52232192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970511534603811, + "loss": 4.4799, + "theoretical_loss": 5.24256696664807, + "tokens_seen": 52297728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970411233701103, + "loss": 4.1907, + "theoretical_loss": 5.24157349257378, + "tokens_seen": 52363264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 40418, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.979517936706543, + "objective/train/theoretical_loss": 5.240581608773973, + "objective/train/tokens_used": 72888800, + "theoretical_loss": 5.240581608773973, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970310932798396, + "loss": 4.0593, + "theoretical_loss": 5.240581608773973, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970210631895686, + "loss": 4.2029, + "theoretical_loss": 5.239591310719987, + "tokens_seen": 52494336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970110330992979, + "loss": 4.0965, + "theoretical_loss": 5.238602593901694, + "tokens_seen": 52559872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970010030090271, + "loss": 4.0066, + "theoretical_loss": 5.23761545382739, + "tokens_seen": 52625408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969909729187563, + "loss": 4.1287, + "theoretical_loss": 5.236629886023708, + "tokens_seen": 52690944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969809428284855, + "loss": 4.1441, + "theoretical_loss": 5.235645886035517, + "tokens_seen": 52756480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969709127382147, + "loss": 4.2684, + "theoretical_loss": 5.2346634494258195, + "tokens_seen": 52822016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969608826479438, + "loss": 4.1129, + "theoretical_loss": 5.233682571775664, + "tokens_seen": 52887552 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496950852557673, + "loss": 4.44, + "theoretical_loss": 5.232703248684045, + "tokens_seen": 52953088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969408224674022, + "loss": 4.3805, + "theoretical_loss": 5.231725475767806, + "tokens_seen": 53018624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969307923771314, + "loss": 4.012, + "theoretical_loss": 5.2307492486615494, + "tokens_seen": 53084160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969207622868606, + "loss": 4.3914, + "theoretical_loss": 5.229774563017541, + "tokens_seen": 53149696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969107321965897, + "loss": 4.2825, + "theoretical_loss": 5.2288014145056145, + "tokens_seen": 53215232 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496900702106319, + "loss": 3.9345, + "theoretical_loss": 5.227829798813083, + "tokens_seen": 53280768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968906720160481, + "loss": 4.1139, + "theoretical_loss": 5.2268597116446465, + "tokens_seen": 53346304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968806419257774, + "loss": 4.2213, + "theoretical_loss": 5.225891148722296, + "tokens_seen": 53411840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968706118355065, + "loss": 4.1851, + "theoretical_loss": 5.224924105785227, + "tokens_seen": 53477376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968605817452358, + "loss": 4.1925, + "theoretical_loss": 5.22395857858975, + "tokens_seen": 53542912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968505516549649, + "loss": 3.8839, + "theoretical_loss": 5.222994562909199, + "tokens_seen": 53608448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968405215646941, + "loss": 4.1607, + "theoretical_loss": 5.222032054533841, + "tokens_seen": 53673984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968304914744233, + "loss": 4.2475, + "theoretical_loss": 5.2210710492707895, + "tokens_seen": 53739520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968204613841525, + "loss": 3.881, + "theoretical_loss": 5.220111542943918, + "tokens_seen": 53805056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968104312938817, + "loss": 4.0171, + "theoretical_loss": 5.21915353139377, + "tokens_seen": 53870592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968004012036108, + "loss": 4.0926, + "theoretical_loss": 5.218197010477477, + "tokens_seen": 53936128 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049679037111334, + "loss": 4.3292, + "theoretical_loss": 5.21724197606866, + "tokens_seen": 54001664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 41741, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.073785305023193, + "objective/train/theoretical_loss": 5.216288424057362, + "objective/train/tokens_used": 74527200, + "theoretical_loss": 5.216288424057362, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967803410230692, + "loss": 4.2491, + "theoretical_loss": 5.216288424057362, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967703109327984, + "loss": 4.2446, + "theoretical_loss": 5.21533635034995, + "tokens_seen": 54132736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967602808425276, + "loss": 4.2944, + "theoretical_loss": 5.214385750869031, + "tokens_seen": 54198272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967502507522568, + "loss": 4.1296, + "theoretical_loss": 5.213436621553377, + "tokens_seen": 54263808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967402206619859, + "loss": 4.17, + "theoretical_loss": 5.212488958357831, + "tokens_seen": 54329344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967301905717151, + "loss": 4.0717, + "theoretical_loss": 5.211542757253229, + "tokens_seen": 54394880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967201604814444, + "loss": 4.4436, + "theoretical_loss": 5.210598014226319, + "tokens_seen": 54460416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967101303911735, + "loss": 4.1394, + "theoretical_loss": 5.209654725279678, + "tokens_seen": 54525952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967001003009028, + "loss": 4.4479, + "theoretical_loss": 5.208712886431626, + "tokens_seen": 54591488 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496690070210632, + "loss": 3.9774, + "theoretical_loss": 5.207772493716154, + "tokens_seen": 54657024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966800401203611, + "loss": 4.0348, + "theoretical_loss": 5.206833543182833, + "tokens_seen": 54722560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966700100300903, + "loss": 4.1275, + "theoretical_loss": 5.205896030896748, + "tokens_seen": 54788096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966599799398195, + "loss": 4.034, + "theoretical_loss": 5.2049599529384025, + "tokens_seen": 54853632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966499498495487, + "loss": 4.2901, + "theoretical_loss": 5.204025305403653, + "tokens_seen": 54919168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966399197592779, + "loss": 4.0859, + "theoretical_loss": 5.2030920844036235, + "tokens_seen": 54984704 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496629889669007, + "loss": 4.2186, + "theoretical_loss": 5.202160286064628, + "tokens_seen": 55050240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966198595787362, + "loss": 4.1753, + "theoretical_loss": 5.2012299065281, + "tokens_seen": 55115776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966098294884654, + "loss": 4.2037, + "theoretical_loss": 5.2003009419505055, + "tokens_seen": 55181312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965997993981946, + "loss": 4.3058, + "theoretical_loss": 5.199373388503272, + "tokens_seen": 55246848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965897693079238, + "loss": 4.3641, + "theoretical_loss": 5.198447242372716, + "tokens_seen": 55312384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965797392176529, + "loss": 4.1662, + "theoretical_loss": 5.197522499759961, + "tokens_seen": 55377920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965697091273821, + "loss": 4.1908, + "theoretical_loss": 5.196599156880863, + "tokens_seen": 55443456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965596790371113, + "loss": 3.9863, + "theoretical_loss": 5.195677209965942, + "tokens_seen": 55508992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965496489468405, + "loss": 4.5222, + "theoretical_loss": 5.194756655260305, + "tokens_seen": 55574528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965396188565698, + "loss": 3.9698, + "theoretical_loss": 5.193837489023565, + "tokens_seen": 55640064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 42409, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.305491924285889, + "objective/train/theoretical_loss": 5.192919707529784, + "objective/train/tokens_used": 76165600, + "theoretical_loss": 5.192919707529784, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965295887662988, + "loss": 4.121, + "theoretical_loss": 5.192919707529784, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965195586760281, + "loss": 4.108, + "theoretical_loss": 5.1920033070673846, + "tokens_seen": 55771136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965095285857573, + "loss": 3.9647, + "theoretical_loss": 5.191088283939086, + "tokens_seen": 55836672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964994984954865, + "loss": 3.9453, + "theoretical_loss": 5.190174634461828, + "tokens_seen": 55902208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964894684052157, + "loss": 3.9193, + "theoretical_loss": 5.189262354966708, + "tokens_seen": 55967744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964794383149449, + "loss": 4.0925, + "theoretical_loss": 5.188351441798897, + "tokens_seen": 56033280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496469408224674, + "loss": 4.1099, + "theoretical_loss": 5.187441891317581, + "tokens_seen": 56098816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964593781344032, + "loss": 4.0055, + "theoretical_loss": 5.186533699895888, + "tokens_seen": 56164352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964493480441324, + "loss": 4.1547, + "theoretical_loss": 5.18562686392081, + "tokens_seen": 56229888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964393179538616, + "loss": 4.1385, + "theoretical_loss": 5.184721379793145, + "tokens_seen": 56295424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964292878635908, + "loss": 4.1137, + "theoretical_loss": 5.183817243927425, + "tokens_seen": 56360960 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049641925777332, + "loss": 3.988, + "theoretical_loss": 5.182914452751846, + "tokens_seen": 56426496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964092276830491, + "loss": 3.9087, + "theoretical_loss": 5.1820130027081985, + "tokens_seen": 56492032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963991975927783, + "loss": 4.1078, + "theoretical_loss": 5.181112890251804, + "tokens_seen": 56557568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963891675025075, + "loss": 3.9657, + "theoretical_loss": 5.180214111851447, + "tokens_seen": 56623104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963791374122367, + "loss": 3.8969, + "theoretical_loss": 5.179316663989305, + "tokens_seen": 56688640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963691073219659, + "loss": 4.0938, + "theoretical_loss": 5.178420543160889, + "tokens_seen": 56754176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963590772316951, + "loss": 4.0717, + "theoretical_loss": 5.177525745874968, + "tokens_seen": 56819712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963490471414242, + "loss": 4.0293, + "theoretical_loss": 5.176632268653516, + "tokens_seen": 56885248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963390170511535, + "loss": 4.0914, + "theoretical_loss": 5.175740108031633, + "tokens_seen": 56950784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963289869608827, + "loss": 4.3386, + "theoretical_loss": 5.174849260557494, + "tokens_seen": 57016320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963189568706119, + "loss": 4.1438, + "theoretical_loss": 5.173959722792272, + "tokens_seen": 57081856 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496308926780341, + "loss": 4.1502, + "theoretical_loss": 5.173071491310086, + "tokens_seen": 57147392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962988966900702, + "loss": 3.9246, + "theoretical_loss": 5.172184562697928, + "tokens_seen": 57212928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962888665997994, + "loss": 4.0249, + "theoretical_loss": 5.171298933555606, + "tokens_seen": 57278464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 43695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.649152994155884, + "objective/train/theoretical_loss": 5.170414600495677, + "objective/train/tokens_used": 77804000, + "theoretical_loss": 5.170414600495677, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962788365095286, + "loss": 4.0093, + "theoretical_loss": 5.170414600495677, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962688064192578, + "loss": 4.1786, + "theoretical_loss": 5.16953156014339, + "tokens_seen": 57409536 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496258776328987, + "loss": 4.0607, + "theoretical_loss": 5.1686498091366175, + "tokens_seen": 57475072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962487462387161, + "loss": 3.6266, + "theoretical_loss": 5.1677693441258, + "tokens_seen": 57540608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962387161484453, + "loss": 3.8942, + "theoretical_loss": 5.16689016177388, + "tokens_seen": 57606144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962286860581746, + "loss": 3.9167, + "theoretical_loss": 5.166012258756243, + "tokens_seen": 57671680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962186559679037, + "loss": 4.2399, + "theoretical_loss": 5.16513563176066, + "tokens_seen": 57737216 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496208625877633, + "loss": 3.7377, + "theoretical_loss": 5.164260277487221, + "tokens_seen": 57802752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961985957873621, + "loss": 4.0589, + "theoretical_loss": 5.163386192648282, + "tokens_seen": 57868288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961885656970913, + "loss": 4.2436, + "theoretical_loss": 5.162513373968403, + "tokens_seen": 57933824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961785356068205, + "loss": 4.2688, + "theoretical_loss": 5.1616418181842825, + "tokens_seen": 57999360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961685055165497, + "loss": 3.9842, + "theoretical_loss": 5.160771522044712, + "tokens_seen": 58064896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961584754262789, + "loss": 4.202, + "theoretical_loss": 5.159902482310506, + "tokens_seen": 58130432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961484453360081, + "loss": 4.0252, + "theoretical_loss": 5.15903469575445, + "tokens_seen": 58195968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961384152457372, + "loss": 4.0653, + "theoretical_loss": 5.1581681591612405, + "tokens_seen": 58261504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961283851554664, + "loss": 4.1853, + "theoretical_loss": 5.15730286932743, + "tokens_seen": 58327040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961183550651956, + "loss": 4.0287, + "theoretical_loss": 5.156438823061366, + "tokens_seen": 58392576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961083249749248, + "loss": 3.8706, + "theoretical_loss": 5.15557601718314, + "tokens_seen": 58458112 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496098294884654, + "loss": 3.4818, + "theoretical_loss": 5.154714448524525, + "tokens_seen": 58523648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960882647943831, + "loss": 4.1847, + "theoretical_loss": 5.153854113928924, + "tokens_seen": 58589184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960782347041123, + "loss": 4.0123, + "theoretical_loss": 5.152995010251313, + "tokens_seen": 58654720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960682046138415, + "loss": 3.8546, + "theoretical_loss": 5.1521371343581865, + "tokens_seen": 58720256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960581745235707, + "loss": 3.9651, + "theoretical_loss": 5.1512804831275, + "tokens_seen": 58785792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960481444333, + "loss": 3.9367, + "theoretical_loss": 5.1504250534486165, + "tokens_seen": 58851328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496038114343029, + "loss": 4.3063, + "theoretical_loss": 5.149570842222257, + "tokens_seen": 58916864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 44395, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.063347339630127, + "objective/train/theoretical_loss": 5.1487178463604355, + "objective/train/tokens_used": 79442400, + "theoretical_loss": 5.1487178463604355, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960280842527583, + "loss": 3.7506, + "theoretical_loss": 5.1487178463604355, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960180541624875, + "loss": 3.9872, + "theoretical_loss": 5.147866062786418, + "tokens_seen": 59047936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960080240722167, + "loss": 3.9911, + "theoretical_loss": 5.147015488434661, + "tokens_seen": 59113472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959979939819459, + "loss": 3.8278, + "theoretical_loss": 5.14616612025076, + "tokens_seen": 59179008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959879638916751, + "loss": 3.9528, + "theoretical_loss": 5.145317955191397, + "tokens_seen": 59244544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959779338014042, + "loss": 3.7919, + "theoretical_loss": 5.144470990224288, + "tokens_seen": 59310080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959679037111334, + "loss": 3.9634, + "theoretical_loss": 5.143625222328134, + "tokens_seen": 59375616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959578736208626, + "loss": 4.096, + "theoretical_loss": 5.142780648492563, + "tokens_seen": 59441152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959478435305918, + "loss": 3.9749, + "theoretical_loss": 5.141937265718084, + "tokens_seen": 59506688 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495937813440321, + "loss": 4.0214, + "theoretical_loss": 5.141095071016032, + "tokens_seen": 59572224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959277833500501, + "loss": 3.9143, + "theoretical_loss": 5.14025406140852, + "tokens_seen": 59637760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959177532597793, + "loss": 4.0642, + "theoretical_loss": 5.139414233928385, + "tokens_seen": 59703296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959077231695085, + "loss": 3.9695, + "theoretical_loss": 5.138575585619142, + "tokens_seen": 59768832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958976930792377, + "loss": 4.1946, + "theoretical_loss": 5.13773811353493, + "tokens_seen": 59834368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958876629889669, + "loss": 4.0784, + "theoretical_loss": 5.1369018147404635, + "tokens_seen": 59899904 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495877632898696, + "loss": 4.0826, + "theoretical_loss": 5.136066686310986, + "tokens_seen": 59965440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958676028084253, + "loss": 4.0869, + "theoretical_loss": 5.135232725332212, + "tokens_seen": 60030976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958575727181544, + "loss": 3.9758, + "theoretical_loss": 5.134399928900292, + "tokens_seen": 60096512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958475426278837, + "loss": 4.0519, + "theoretical_loss": 5.13356829412175, + "tokens_seen": 60162048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958375125376129, + "loss": 4.1545, + "theoretical_loss": 5.132737818113444, + "tokens_seen": 60227584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958274824473421, + "loss": 4.2355, + "theoretical_loss": 5.131908498002513, + "tokens_seen": 60293120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958174523570712, + "loss": 3.8189, + "theoretical_loss": 5.131080330926334, + "tokens_seen": 60358656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958074222668004, + "loss": 4.1869, + "theoretical_loss": 5.130253314032469, + "tokens_seen": 60424192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957973921765296, + "loss": 3.728, + "theoretical_loss": 5.1294274444786225, + "tokens_seen": 60489728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957873620862588, + "loss": 4.1881, + "theoretical_loss": 5.128602719432592, + "tokens_seen": 60555264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 45571, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.731624603271484, + "objective/train/theoretical_loss": 5.12777913607222, + "objective/train/tokens_used": 81080800, + "theoretical_loss": 5.12777913607222, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495777331995988, + "loss": 4.3579, + "theoretical_loss": 5.12777913607222, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957673019057172, + "loss": 3.8709, + "theoretical_loss": 5.126956691585351, + "tokens_seen": 60686336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957572718154463, + "loss": 3.9854, + "theoretical_loss": 5.1261353831697845, + "tokens_seen": 60751872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957472417251755, + "loss": 4.2274, + "theoretical_loss": 5.125315208033226, + "tokens_seen": 60817408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957372116349047, + "loss": 3.8993, + "theoretical_loss": 5.124496163393244, + "tokens_seen": 60882944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957271815446339, + "loss": 3.7806, + "theoretical_loss": 5.123678246477226, + "tokens_seen": 60948480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957171514543631, + "loss": 3.977, + "theoretical_loss": 5.122861454522329, + "tokens_seen": 61014016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957071213640923, + "loss": 3.9739, + "theoretical_loss": 5.122045784775439, + "tokens_seen": 61079552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956970912738214, + "loss": 3.8256, + "theoretical_loss": 5.1212312344931235, + "tokens_seen": 61145088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956870611835507, + "loss": 4.0096, + "theoretical_loss": 5.120417800941587, + "tokens_seen": 61210624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956770310932798, + "loss": 4.0399, + "theoretical_loss": 5.119605481396629, + "tokens_seen": 61276160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956670010030091, + "loss": 3.9718, + "theoretical_loss": 5.1187942731436, + "tokens_seen": 61341696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956569709127383, + "loss": 3.9982, + "theoretical_loss": 5.117984173477356, + "tokens_seen": 61407232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956469408224674, + "loss": 4.0777, + "theoretical_loss": 5.117175179702215, + "tokens_seen": 61472768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956369107321966, + "loss": 3.9865, + "theoretical_loss": 5.116367289131917, + "tokens_seen": 61538304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956268806419258, + "loss": 4.1915, + "theoretical_loss": 5.115560499089575, + "tokens_seen": 61603840 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495616850551655, + "loss": 3.8813, + "theoretical_loss": 5.1147548069076425, + "tokens_seen": 61669376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956068204613842, + "loss": 3.97, + "theoretical_loss": 5.113950209927859, + "tokens_seen": 61734912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955967903711133, + "loss": 3.5137, + "theoretical_loss": 5.113146705501217, + "tokens_seen": 61800448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955867602808425, + "loss": 3.7782, + "theoretical_loss": 5.112344290987915, + "tokens_seen": 61865984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955767301905717, + "loss": 4.1055, + "theoretical_loss": 5.1115429637573175, + "tokens_seen": 61931520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955667001003009, + "loss": 3.9588, + "theoretical_loss": 5.110742721187915, + "tokens_seen": 61997056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955566700100301, + "loss": 3.8558, + "theoretical_loss": 5.109943560667279, + "tokens_seen": 62062592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955466399197592, + "loss": 3.8969, + "theoretical_loss": 5.109145479592028, + "tokens_seen": 62128128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955366098294884, + "loss": 4.1078, + "theoretical_loss": 5.108348475367774, + "tokens_seen": 62193664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 46598, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.215669631958008, + "objective/train/theoretical_loss": 5.107552545409097, + "objective/train/tokens_used": 82719200, + "theoretical_loss": 5.107552545409097, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955265797392177, + "loss": 4.056, + "theoretical_loss": 5.107552545409097, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955165496489468, + "loss": 3.9227, + "theoretical_loss": 5.106757687139494, + "tokens_seen": 62324736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955065195586761, + "loss": 4.2246, + "theoretical_loss": 5.1059638979913435, + "tokens_seen": 62390272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954964894684052, + "loss": 4.0438, + "theoretical_loss": 5.105171175405864, + "tokens_seen": 62455808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954864593781344, + "loss": 4.0275, + "theoretical_loss": 5.104379516833075, + "tokens_seen": 62521344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954764292878636, + "loss": 4.005, + "theoretical_loss": 5.103588919731757, + "tokens_seen": 62586880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954663991975928, + "loss": 3.937, + "theoretical_loss": 5.1027993815694135, + "tokens_seen": 62652416 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495456369107322, + "loss": 4.1536, + "theoretical_loss": 5.10201089982223, + "tokens_seen": 62717952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954463390170512, + "loss": 3.971, + "theoretical_loss": 5.101223471975038, + "tokens_seen": 62783488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954363089267803, + "loss": 3.9062, + "theoretical_loss": 5.100437095521272, + "tokens_seen": 62849024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954262788365095, + "loss": 3.8866, + "theoretical_loss": 5.099651767962936, + "tokens_seen": 62914560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954162487462387, + "loss": 3.8535, + "theoretical_loss": 5.098867486810562, + "tokens_seen": 62980096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954062186559679, + "loss": 3.7859, + "theoretical_loss": 5.098084249583175, + "tokens_seen": 63045632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953961885656971, + "loss": 3.9093, + "theoretical_loss": 5.097302053808251, + "tokens_seen": 63111168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953861584754263, + "loss": 3.5672, + "theoretical_loss": 5.0965208970216835, + "tokens_seen": 63176704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953761283851555, + "loss": 3.8846, + "theoretical_loss": 5.095740776767743, + "tokens_seen": 63242240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953660982948846, + "loss": 3.9469, + "theoretical_loss": 5.0949616905990425, + "tokens_seen": 63307776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953560682046139, + "loss": 4.2765, + "theoretical_loss": 5.094183636076501, + "tokens_seen": 63373312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953460381143431, + "loss": 4.015, + "theoretical_loss": 5.093406610769302, + "tokens_seen": 63438848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953360080240723, + "loss": 4.2246, + "theoretical_loss": 5.092630612254862, + "tokens_seen": 63504384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953259779338014, + "loss": 3.9173, + "theoretical_loss": 5.091855638118792, + "tokens_seen": 63569920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953159478435306, + "loss": 4.0249, + "theoretical_loss": 5.091081685954863, + "tokens_seen": 63635456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953059177532598, + "loss": 3.7978, + "theoretical_loss": 5.090308753364967, + "tokens_seen": 63700992 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495295887662989, + "loss": 3.8605, + "theoretical_loss": 5.089536837959084, + "tokens_seen": 63766528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952858575727182, + "loss": 3.9657, + "theoretical_loss": 5.0887659373552445, + "tokens_seen": 63832064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 47189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3733479976654053, + "objective/train/theoretical_loss": 5.087996049179497, + "objective/train/tokens_used": 84357600, + "theoretical_loss": 5.087996049179497, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952758274824474, + "loss": 3.6824, + "theoretical_loss": 5.087996049179497, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952657973921765, + "loss": 3.7677, + "theoretical_loss": 5.087227171065869, + "tokens_seen": 63963136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952557673019057, + "loss": 3.9197, + "theoretical_loss": 5.086459300656337, + "tokens_seen": 64028672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952457372116349, + "loss": 3.927, + "theoretical_loss": 5.085692435600783, + "tokens_seen": 64094208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952357071213641, + "loss": 3.8821, + "theoretical_loss": 5.084926573556972, + "tokens_seen": 64159744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952256770310933, + "loss": 3.7931, + "theoretical_loss": 5.084161712190507, + "tokens_seen": 64225280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952156469408225, + "loss": 3.7647, + "theoretical_loss": 5.083397849174801, + "tokens_seen": 64290816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952056168505516, + "loss": 3.9431, + "theoretical_loss": 5.082634982191042, + "tokens_seen": 64356352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951955867602809, + "loss": 3.9173, + "theoretical_loss": 5.081873108928154, + "tokens_seen": 64421888 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049518555667001, + "loss": 4.0204, + "theoretical_loss": 5.081112227082771, + "tokens_seen": 64487424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951755265797393, + "loss": 3.816, + "theoretical_loss": 5.0803523343591985, + "tokens_seen": 64552960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951654964894685, + "loss": 3.8761, + "theoretical_loss": 5.079593428469382, + "tokens_seen": 64618496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951554663991976, + "loss": 4.0756, + "theoretical_loss": 5.078835507132873, + "tokens_seen": 64684032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951454363089268, + "loss": 3.8136, + "theoretical_loss": 5.078078568076798, + "tokens_seen": 64749568 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495135406218656, + "loss": 4.0581, + "theoretical_loss": 5.077322609035821, + "tokens_seen": 64815104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951253761283852, + "loss": 3.8215, + "theoretical_loss": 5.076567627752118, + "tokens_seen": 64880640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951153460381144, + "loss": 3.8355, + "theoretical_loss": 5.075813621975337, + "tokens_seen": 64946176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951053159478435, + "loss": 3.9719, + "theoretical_loss": 5.075060589462572, + "tokens_seen": 65011712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950952858575727, + "loss": 3.7383, + "theoretical_loss": 5.074308527978326, + "tokens_seen": 65077248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950852557673019, + "loss": 3.8755, + "theoretical_loss": 5.073557435294486, + "tokens_seen": 65142784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950752256770311, + "loss": 3.6252, + "theoretical_loss": 5.07280730919028, + "tokens_seen": 65208320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950651955867603, + "loss": 3.7529, + "theoretical_loss": 5.072058147452257, + "tokens_seen": 65273856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950551654964894, + "loss": 3.679, + "theoretical_loss": 5.071309947874248, + "tokens_seen": 65339392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950451354062186, + "loss": 3.5894, + "theoretical_loss": 5.070562708257339, + "tokens_seen": 65404928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950351053159479, + "loss": 3.9951, + "theoretical_loss": 5.069816426409837, + "tokens_seen": 65470464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 48433, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.8971009254455566, + "objective/train/theoretical_loss": 5.069071100147241, + "objective/train/tokens_used": 85996000, + "theoretical_loss": 5.069071100147241, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495025075225677, + "loss": 3.9133, + "theoretical_loss": 5.069071100147241, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950150451354063, + "loss": 4.2034, + "theoretical_loss": 5.068326727292211, + "tokens_seen": 65601536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950050150451354, + "loss": 3.7703, + "theoretical_loss": 5.067583305674537, + "tokens_seen": 65667072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949949849548646, + "loss": 3.8678, + "theoretical_loss": 5.066840833131106, + "tokens_seen": 65732608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949849548645938, + "loss": 3.8256, + "theoretical_loss": 5.066099307505878, + "tokens_seen": 65798144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494974924774323, + "loss": 3.6242, + "theoretical_loss": 5.06535872664985, + "tokens_seen": 65863680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949648946840522, + "loss": 3.9033, + "theoretical_loss": 5.064619088421029, + "tokens_seen": 65929216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949548645937814, + "loss": 3.5217, + "theoretical_loss": 5.0638803906844, + "tokens_seen": 65994752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949448345035105, + "loss": 3.8469, + "theoretical_loss": 5.0631426313119, + "tokens_seen": 66060288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949348044132397, + "loss": 3.7472, + "theoretical_loss": 5.062405808182383, + "tokens_seen": 66125824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949247743229689, + "loss": 3.4883, + "theoretical_loss": 5.061669919181599, + "tokens_seen": 66191360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949147442326981, + "loss": 3.8441, + "theoretical_loss": 5.060934962202155, + "tokens_seen": 66256896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949047141424273, + "loss": 3.9786, + "theoretical_loss": 5.060200935143492, + "tokens_seen": 66322432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948946840521565, + "loss": 4.0608, + "theoretical_loss": 5.059467835911857, + "tokens_seen": 66387968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948846539618856, + "loss": 3.7521, + "theoretical_loss": 5.058735662420268, + "tokens_seen": 66453504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948746238716148, + "loss": 3.6167, + "theoretical_loss": 5.0580044125884935, + "tokens_seen": 66519040 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494864593781344, + "loss": 3.76, + "theoretical_loss": 5.057274084343018, + "tokens_seen": 66584576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948545636910733, + "loss": 3.8083, + "theoretical_loss": 5.056544675617017, + "tokens_seen": 66650112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948445336008024, + "loss": 3.7601, + "theoretical_loss": 5.055816184350326, + "tokens_seen": 66715648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948345035105316, + "loss": 3.8069, + "theoretical_loss": 5.055088608489417, + "tokens_seen": 66781184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948244734202607, + "loss": 3.3711, + "theoretical_loss": 5.054361945987363, + "tokens_seen": 66846720 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049481444332999, + "loss": 3.3429, + "theoretical_loss": 5.05363619480382, + "tokens_seen": 66912256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948044132397192, + "loss": 4.0226, + "theoretical_loss": 5.052911352904992, + "tokens_seen": 66977792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947943831494484, + "loss": 3.7436, + "theoretical_loss": 5.052187418263609, + "tokens_seen": 67043328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947843530591776, + "loss": 3.8333, + "theoretical_loss": 5.051464388858893, + "tokens_seen": 67108864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 48877, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7897393703460693, + "objective/train/theoretical_loss": 5.050742262676536, + "objective/train/tokens_used": 87634400, + "theoretical_loss": 5.050742262676536, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947743229689067, + "loss": 3.7583, + "theoretical_loss": 5.050742262676536, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947642928786359, + "loss": 4.0895, + "theoretical_loss": 5.050021037708673, + "tokens_seen": 67239936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947542627883651, + "loss": 3.642, + "theoretical_loss": 5.049300711953854, + "tokens_seen": 67305472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947442326980943, + "loss": 3.7434, + "theoretical_loss": 5.048581283417015, + "tokens_seen": 67371008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947342026078235, + "loss": 3.7182, + "theoretical_loss": 5.047862750109456, + "tokens_seen": 67436544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947241725175527, + "loss": 3.9326, + "theoretical_loss": 5.047145110048811, + "tokens_seen": 67502080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947141424272818, + "loss": 3.638, + "theoretical_loss": 5.046428361259022, + "tokens_seen": 67567616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947041123370111, + "loss": 3.7471, + "theoretical_loss": 5.045712501770317, + "tokens_seen": 67633152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946940822467402, + "loss": 3.699, + "theoretical_loss": 5.0449975296191765, + "tokens_seen": 67698688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946840521564695, + "loss": 3.8026, + "theoretical_loss": 5.0442834428483145, + "tokens_seen": 67764224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946740220661987, + "loss": 3.7185, + "theoretical_loss": 5.043570239506648, + "tokens_seen": 67829760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946639919759278, + "loss": 3.8962, + "theoretical_loss": 5.042857917649275, + "tokens_seen": 67895296 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494653961885657, + "loss": 3.7692, + "theoretical_loss": 5.042146475337448, + "tokens_seen": 67960832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946439317953862, + "loss": 3.5141, + "theoretical_loss": 5.041435910638544, + "tokens_seen": 68026368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946339017051154, + "loss": 3.8396, + "theoretical_loss": 5.040726221626048, + "tokens_seen": 68091904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946238716148446, + "loss": 3.8633, + "theoretical_loss": 5.040017406379521, + "tokens_seen": 68157440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946138415245737, + "loss": 3.6866, + "theoretical_loss": 5.0393094629845745, + "tokens_seen": 68222976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946038114343029, + "loss": 3.5706, + "theoretical_loss": 5.038602389532853, + "tokens_seen": 68288512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945937813440321, + "loss": 3.4773, + "theoretical_loss": 5.037896184122, + "tokens_seen": 68354048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945837512537613, + "loss": 3.7606, + "theoretical_loss": 5.037190844855641, + "tokens_seen": 68419584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945737211634905, + "loss": 3.8062, + "theoretical_loss": 5.036486369843355, + "tokens_seen": 68485120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945636910732196, + "loss": 3.6164, + "theoretical_loss": 5.03578275720065, + "tokens_seen": 68550656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945536609829488, + "loss": 3.7046, + "theoretical_loss": 5.03508000504894, + "tokens_seen": 68616192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945436308926781, + "loss": 3.7478, + "theoretical_loss": 5.034378111515521, + "tokens_seen": 68681728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945336008024072, + "loss": 3.663, + "theoretical_loss": 5.0336770747335455, + "tokens_seen": 68747264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 49922, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9727299213409424, + "objective/train/theoretical_loss": 5.032976892842002, + "objective/train/tokens_used": 89272800, + "theoretical_loss": 5.032976892842002, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945235707121365, + "loss": 3.6731, + "theoretical_loss": 5.032976892842002, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945135406218656, + "loss": 3.8125, + "theoretical_loss": 5.032277563985687, + "tokens_seen": 68878336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945035105315948, + "loss": 3.8008, + "theoretical_loss": 5.031579086315183, + "tokens_seen": 68943872 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494493480441324, + "loss": 3.9551, + "theoretical_loss": 5.030881457986837, + "tokens_seen": 69009408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944834503510532, + "loss": 3.843, + "theoretical_loss": 5.030184677162734, + "tokens_seen": 69074944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944734202607824, + "loss": 3.5448, + "theoretical_loss": 5.029488742010676, + "tokens_seen": 69140480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944633901705116, + "loss": 3.8376, + "theoretical_loss": 5.028793650704154, + "tokens_seen": 69206016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944533600802407, + "loss": 3.9125, + "theoretical_loss": 5.028099401422336, + "tokens_seen": 69271552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944433299899699, + "loss": 3.6831, + "theoretical_loss": 5.0274059923500305, + "tokens_seen": 69337088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944332998996991, + "loss": 3.7007, + "theoretical_loss": 5.026713421677673, + "tokens_seen": 69402624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944232698094283, + "loss": 3.6241, + "theoretical_loss": 5.0260216876013, + "tokens_seen": 69468160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944132397191575, + "loss": 3.7006, + "theoretical_loss": 5.025330788322526, + "tokens_seen": 69533696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944032096288867, + "loss": 3.9117, + "theoretical_loss": 5.024640722048522, + "tokens_seen": 69599232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943931795386158, + "loss": 3.4086, + "theoretical_loss": 5.023951486991994, + "tokens_seen": 69664768 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494383149448345, + "loss": 3.9368, + "theoretical_loss": 5.023263081371157, + "tokens_seen": 69730304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943731193580742, + "loss": 3.8784, + "theoretical_loss": 5.022575503409717, + "tokens_seen": 69795840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943630892678035, + "loss": 3.941, + "theoretical_loss": 5.021888751336848, + "tokens_seen": 69861376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943530591775326, + "loss": 3.8625, + "theoretical_loss": 5.02120282338717, + "tokens_seen": 69926912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943430290872618, + "loss": 3.6762, + "theoretical_loss": 5.020517717800723, + "tokens_seen": 69992448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943329989969909, + "loss": 3.4889, + "theoretical_loss": 5.019833432822953, + "tokens_seen": 70057984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943229689067202, + "loss": 3.5122, + "theoretical_loss": 5.019149966704683, + "tokens_seen": 70123520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943129388164494, + "loss": 3.6293, + "theoretical_loss": 5.018467317702097, + "tokens_seen": 70189056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943029087261786, + "loss": 3.5085, + "theoretical_loss": 5.017785484076715, + "tokens_seen": 70254592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942928786359078, + "loss": 3.6348, + "theoretical_loss": 5.017104464095375, + "tokens_seen": 70320128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828485456369, + "loss": 3.6907, + "theoretical_loss": 5.0164242560302075, + "tokens_seen": 70385664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 51260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.9014105796813965, + "objective/train/theoretical_loss": 5.015744858158618, + "objective/train/tokens_used": 90911200, + "theoretical_loss": 5.015744858158618, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942728184553661, + "loss": 3.8765, + "theoretical_loss": 5.015744858158618, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942627883650953, + "loss": 3.8516, + "theoretical_loss": 5.015066268763264, + "tokens_seen": 70516736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942527582748245, + "loss": 3.6519, + "theoretical_loss": 5.014388486132036, + "tokens_seen": 70582272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942427281845537, + "loss": 3.9132, + "theoretical_loss": 5.0137115085580355, + "tokens_seen": 70647808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942326980942828, + "loss": 3.8834, + "theoretical_loss": 5.013035334339554, + "tokens_seen": 70713344 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494222668004012, + "loss": 3.8794, + "theoretical_loss": 5.012359961780053, + "tokens_seen": 70778880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942126379137412, + "loss": 3.8483, + "theoretical_loss": 5.011685389188142, + "tokens_seen": 70844416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942026078234704, + "loss": 3.4989, + "theoretical_loss": 5.011011614877562, + "tokens_seen": 70909952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941925777331996, + "loss": 3.7672, + "theoretical_loss": 5.010338637167162, + "tokens_seen": 70975488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941825476429289, + "loss": 3.6359, + "theoretical_loss": 5.009666454380877, + "tokens_seen": 71041024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941725175526579, + "loss": 3.8619, + "theoretical_loss": 5.008995064847716, + "tokens_seen": 71106560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941624874623872, + "loss": 3.8008, + "theoretical_loss": 5.0083244669017315, + "tokens_seen": 71172096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941524573721163, + "loss": 3.9147, + "theoretical_loss": 5.0076546588820054, + "tokens_seen": 71237632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941424272818456, + "loss": 3.3797, + "theoretical_loss": 5.00698563913263, + "tokens_seen": 71303168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941323971915748, + "loss": 3.617, + "theoretical_loss": 5.006317406002687, + "tokens_seen": 71368704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941223671013039, + "loss": 3.5556, + "theoretical_loss": 5.0056499578462255, + "tokens_seen": 71434240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941123370110331, + "loss": 3.4481, + "theoretical_loss": 5.0049832930222475, + "tokens_seen": 71499776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941023069207623, + "loss": 3.826, + "theoretical_loss": 5.004317409894682, + "tokens_seen": 71565312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940922768304915, + "loss": 3.6718, + "theoretical_loss": 5.003652306832372, + "tokens_seen": 71630848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940822467402207, + "loss": 3.8578, + "theoretical_loss": 5.002987982209052, + "tokens_seen": 71696384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940722166499498, + "loss": 3.758, + "theoretical_loss": 5.00232443440333, + "tokens_seen": 71761920 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494062186559679, + "loss": 3.4768, + "theoretical_loss": 5.001661661798666, + "tokens_seen": 71827456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940521564694082, + "loss": 3.6442, + "theoretical_loss": 5.000999662783359, + "tokens_seen": 71892992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940421263791374, + "loss": 3.7435, + "theoretical_loss": 5.0003384357505185, + "tokens_seen": 71958528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940320962888666, + "loss": 3.8745, + "theoretical_loss": 4.999677979098056, + "tokens_seen": 72024064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 51878, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4065515995025635, + "objective/train/theoretical_loss": 4.999018291228659, + "objective/train/tokens_used": 92549600, + "theoretical_loss": 4.999018291228659, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940220661985958, + "loss": 3.5808, + "theoretical_loss": 4.999018291228659, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940120361083249, + "loss": 3.8043, + "theoretical_loss": 4.998359370549778, + "tokens_seen": 72155136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940020060180542, + "loss": 3.649, + "theoretical_loss": 4.997701215473601, + "tokens_seen": 72220672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939919759277834, + "loss": 3.6107, + "theoretical_loss": 4.997043824417044, + "tokens_seen": 72286208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939819458375126, + "loss": 3.6793, + "theoretical_loss": 4.996387195801723, + "tokens_seen": 72351744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939719157472418, + "loss": 3.6529, + "theoretical_loss": 4.995731328053944, + "tokens_seen": 72417280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493961885656971, + "loss": 3.7394, + "theoretical_loss": 4.995076219604682, + "tokens_seen": 72482816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939518555667001, + "loss": 3.6256, + "theoretical_loss": 4.994421868889559, + "tokens_seen": 72548352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939418254764293, + "loss": 3.6209, + "theoretical_loss": 4.9937682743488345, + "tokens_seen": 72613888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939317953861585, + "loss": 3.5004, + "theoretical_loss": 4.993115434427376, + "tokens_seen": 72679424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939217652958877, + "loss": 3.5103, + "theoretical_loss": 4.992463347574654, + "tokens_seen": 72744960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939117352056169, + "loss": 3.7514, + "theoretical_loss": 4.991812012244717, + "tokens_seen": 72810496 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493901705115346, + "loss": 3.5319, + "theoretical_loss": 4.99116142689617, + "tokens_seen": 72876032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938916750250752, + "loss": 3.8628, + "theoretical_loss": 4.990511589992169, + "tokens_seen": 72941568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938816449348044, + "loss": 4.0163, + "theoretical_loss": 4.989862500000392, + "tokens_seen": 73007104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938716148445337, + "loss": 3.7264, + "theoretical_loss": 4.989214155393025, + "tokens_seen": 73072640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938615847542628, + "loss": 3.9026, + "theoretical_loss": 4.988566554646752, + "tokens_seen": 73138176 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493851554663992, + "loss": 3.4925, + "theoretical_loss": 4.9879196962427255, + "tokens_seen": 73203712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938415245737211, + "loss": 3.5623, + "theoretical_loss": 4.9872735786665565, + "tokens_seen": 73269248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938314944834504, + "loss": 3.415, + "theoretical_loss": 4.986628200408299, + "tokens_seen": 73334784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938214643931796, + "loss": 3.5253, + "theoretical_loss": 4.985983559962429, + "tokens_seen": 73400320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938114343029088, + "loss": 3.5016, + "theoretical_loss": 4.985339655827827, + "tokens_seen": 73465856 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493801404212638, + "loss": 3.8156, + "theoretical_loss": 4.984696486507767, + "tokens_seen": 73531392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937913741223671, + "loss": 3.7367, + "theoretical_loss": 4.984054050509893, + "tokens_seen": 73596928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937813440320963, + "loss": 3.6885, + "theoretical_loss": 4.983412346346206, + "tokens_seen": 73662464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 53017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.889392137527466, + "objective/train/theoretical_loss": 4.982771372533048, + "objective/train/tokens_used": 94188000, + "theoretical_loss": 4.982771372533048, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937713139418255, + "loss": 3.6721, + "theoretical_loss": 4.982771372533048, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937612838515547, + "loss": 3.9426, + "theoretical_loss": 4.982131127591083, + "tokens_seen": 73793536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937512537612839, + "loss": 3.6135, + "theoretical_loss": 4.981491610045282, + "tokens_seen": 73859072 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493741223671013, + "loss": 3.6257, + "theoretical_loss": 4.980852818424905, + "tokens_seen": 73924608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937311935807422, + "loss": 3.6675, + "theoretical_loss": 4.980214751263491, + "tokens_seen": 73990144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937211634904714, + "loss": 3.6453, + "theoretical_loss": 4.979577407098831, + "tokens_seen": 74055680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937111334002006, + "loss": 3.7323, + "theoretical_loss": 4.978940784472962, + "tokens_seen": 74121216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937011033099298, + "loss": 3.4667, + "theoretical_loss": 4.978304881932145, + "tokens_seen": 74186752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936910732196591, + "loss": 3.9063, + "theoretical_loss": 4.97766969802685, + "tokens_seen": 74252288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936810431293881, + "loss": 3.3322, + "theoretical_loss": 4.977035231311745, + "tokens_seen": 74317824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936710130391174, + "loss": 3.6439, + "theoretical_loss": 4.976401480345672, + "tokens_seen": 74383360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936609829488465, + "loss": 3.5552, + "theoretical_loss": 4.975768443691636, + "tokens_seen": 74448896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936509528585758, + "loss": 3.5095, + "theoretical_loss": 4.97513611991679, + "tokens_seen": 74514432 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493640922768305, + "loss": 3.7738, + "theoretical_loss": 4.974504507592419, + "tokens_seen": 74579968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936308926780341, + "loss": 3.8278, + "theoretical_loss": 4.973873605293922, + "tokens_seen": 74645504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936208625877633, + "loss": 3.7248, + "theoretical_loss": 4.973243411600797, + "tokens_seen": 74711040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936108324974925, + "loss": 3.8762, + "theoretical_loss": 4.9726139250966295, + "tokens_seen": 74776576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936008024072217, + "loss": 3.6798, + "theoretical_loss": 4.971985144369073, + "tokens_seen": 74842112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935907723169509, + "loss": 3.6282, + "theoretical_loss": 4.9713570680098345, + "tokens_seen": 74907648 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049358074222668, + "loss": 3.7215, + "theoretical_loss": 4.970729694614662, + "tokens_seen": 74973184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935707121364092, + "loss": 3.5638, + "theoretical_loss": 4.970103022783325, + "tokens_seen": 75038720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935606820461384, + "loss": 3.5533, + "theoretical_loss": 4.969477051119606, + "tokens_seen": 75104256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935506519558676, + "loss": 3.496, + "theoretical_loss": 4.968851778231276, + "tokens_seen": 75169792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935406218655968, + "loss": 3.6459, + "theoretical_loss": 4.9682272027300876, + "tokens_seen": 75235328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493530591775326, + "loss": 3.5601, + "theoretical_loss": 4.96760332323176, + "tokens_seen": 75300864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 53591, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.8066093921661377, + "objective/train/theoretical_loss": 4.966980138355957, + "objective/train/tokens_used": 95826400, + "theoretical_loss": 4.966980138355957, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935205616850551, + "loss": 3.7377, + "theoretical_loss": 4.966980138355957, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935105315947844, + "loss": 3.6051, + "theoretical_loss": 4.966357646726282, + "tokens_seen": 75431936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935005015045135, + "loss": 3.5847, + "theoretical_loss": 4.965735846970255, + "tokens_seen": 75497472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934904714142428, + "loss": 3.7027, + "theoretical_loss": 4.965114737719302, + "tokens_seen": 75563008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934804413239719, + "loss": 3.648, + "theoretical_loss": 4.964494317608743, + "tokens_seen": 75628544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934704112337011, + "loss": 3.4989, + "theoretical_loss": 4.963874585277772, + "tokens_seen": 75694080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934603811434303, + "loss": 3.8889, + "theoretical_loss": 4.963255539369445, + "tokens_seen": 75759616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934503510531595, + "loss": 3.5961, + "theoretical_loss": 4.9626371785306675, + "tokens_seen": 75825152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934403209628887, + "loss": 3.4261, + "theoretical_loss": 4.962019501412178, + "tokens_seen": 75890688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934302908726179, + "loss": 3.7473, + "theoretical_loss": 4.961402506668535, + "tokens_seen": 75956224 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493420260782347, + "loss": 3.4055, + "theoretical_loss": 4.960786192958103, + "tokens_seen": 76021760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934102306920762, + "loss": 3.6034, + "theoretical_loss": 4.960170558943037, + "tokens_seen": 76087296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934002006018054, + "loss": 3.4187, + "theoretical_loss": 4.959555603289269, + "tokens_seen": 76152832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933901705115346, + "loss": 3.6193, + "theoretical_loss": 4.9589413246664975, + "tokens_seen": 76218368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933801404212638, + "loss": 3.794, + "theoretical_loss": 4.958327721748168, + "tokens_seen": 76283904 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493370110330993, + "loss": 3.3982, + "theoretical_loss": 4.957714793211462, + "tokens_seen": 76349440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933600802407221, + "loss": 3.3334, + "theoretical_loss": 4.957102537737284, + "tokens_seen": 76414976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933500501504513, + "loss": 3.6421, + "theoretical_loss": 4.956490954010249, + "tokens_seen": 76480512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933400200601805, + "loss": 3.5119, + "theoretical_loss": 4.9558800407186645, + "tokens_seen": 76546048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933299899699098, + "loss": 3.4933, + "theoretical_loss": 4.95526979655452, + "tokens_seen": 76611584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933199598796389, + "loss": 3.7028, + "theoretical_loss": 4.954660220213473, + "tokens_seen": 76677120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933099297893682, + "loss": 3.7165, + "theoretical_loss": 4.954051310394836, + "tokens_seen": 76742656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932998996990972, + "loss": 3.5507, + "theoretical_loss": 4.953443065801563, + "tokens_seen": 76808192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932898696088265, + "loss": 3.4901, + "theoretical_loss": 4.952835485140236, + "tokens_seen": 76873728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932798395185557, + "loss": 3.2798, + "theoretical_loss": 4.952228567121052, + "tokens_seen": 76939264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 54447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6758010387420654, + "objective/train/theoretical_loss": 4.951622310457807, + "objective/train/tokens_used": 97464800, + "theoretical_loss": 4.951622310457807, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932698094282849, + "loss": 3.4817, + "theoretical_loss": 4.951622310457807, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932597793380141, + "loss": 3.5561, + "theoretical_loss": 4.95101671386789, + "tokens_seen": 77070336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932497492477432, + "loss": 3.6278, + "theoretical_loss": 4.950411776072261, + "tokens_seen": 77135872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932397191574724, + "loss": 3.7609, + "theoretical_loss": 4.949807495795445, + "tokens_seen": 77201408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932296890672016, + "loss": 3.6186, + "theoretical_loss": 4.949203871765516, + "tokens_seen": 77266944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932196589769308, + "loss": 3.6535, + "theoretical_loss": 4.948600902714083, + "tokens_seen": 77332480 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049320962888666, + "loss": 3.4356, + "theoretical_loss": 4.947998587376282, + "tokens_seen": 77398016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931995987963893, + "loss": 3.7607, + "theoretical_loss": 4.947396924490757, + "tokens_seen": 77463552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931895687061183, + "loss": 3.9334, + "theoretical_loss": 4.946795912799653, + "tokens_seen": 77529088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931795386158476, + "loss": 3.4391, + "theoretical_loss": 4.946195551048597, + "tokens_seen": 77594624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931695085255767, + "loss": 3.4097, + "theoretical_loss": 4.945595837986693, + "tokens_seen": 77660160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493159478435306, + "loss": 3.4903, + "theoretical_loss": 4.944996772366503, + "tokens_seen": 77725696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931494483450352, + "loss": 3.5634, + "theoretical_loss": 4.944398352944038, + "tokens_seen": 77791232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931394182547643, + "loss": 3.5448, + "theoretical_loss": 4.943800578478745, + "tokens_seen": 77856768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931293881644935, + "loss": 3.4788, + "theoretical_loss": 4.9432034477334925, + "tokens_seen": 77922304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931193580742227, + "loss": 3.4963, + "theoretical_loss": 4.942606959474562, + "tokens_seen": 77987840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931093279839519, + "loss": 3.4853, + "theoretical_loss": 4.942011112471633, + "tokens_seen": 78053376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930992978936811, + "loss": 3.3058, + "theoretical_loss": 4.94141590549777, + "tokens_seen": 78118912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930892678034102, + "loss": 3.7623, + "theoretical_loss": 4.940821337329414, + "tokens_seen": 78184448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930792377131394, + "loss": 3.324, + "theoretical_loss": 4.940227406746368, + "tokens_seen": 78249984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930692076228686, + "loss": 3.595, + "theoretical_loss": 4.939634112531784, + "tokens_seen": 78315520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930591775325978, + "loss": 3.6571, + "theoretical_loss": 4.939041453472152, + "tokens_seen": 78381056 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493049147442327, + "loss": 3.546, + "theoretical_loss": 4.9384494283572895, + "tokens_seen": 78446592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930391173520562, + "loss": 3.7395, + "theoretical_loss": 4.937858035980327, + "tokens_seen": 78512128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930290872617853, + "loss": 3.5258, + "theoretical_loss": 4.9372672751377, + "tokens_seen": 78577664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 54900, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9137701988220215, + "objective/train/theoretical_loss": 4.936677144629131, + "objective/train/tokens_used": 99103200, + "theoretical_loss": 4.936677144629131, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930190571715146, + "loss": 3.2102, + "theoretical_loss": 4.936677144629131, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930090270812437, + "loss": 3.7118, + "theoretical_loss": 4.936087643257625, + "tokens_seen": 78708736 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492998996990973, + "loss": 3.5876, + "theoretical_loss": 4.935498769829453, + "tokens_seen": 78774272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929889669007021, + "loss": 3.7919, + "theoretical_loss": 4.934910523154139, + "tokens_seen": 78839808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929789368104313, + "loss": 3.8801, + "theoretical_loss": 4.934322902044457, + "tokens_seen": 78905344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929689067201605, + "loss": 3.5096, + "theoretical_loss": 4.933735905316408, + "tokens_seen": 78970880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929588766298897, + "loss": 3.4217, + "theoretical_loss": 4.933149531789217, + "tokens_seen": 79036416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929488465396189, + "loss": 3.6953, + "theoretical_loss": 4.932563780285319, + "tokens_seen": 79101952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929388164493481, + "loss": 3.6487, + "theoretical_loss": 4.931978649630346, + "tokens_seen": 79167488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929287863590773, + "loss": 3.6641, + "theoretical_loss": 4.931394138653117, + "tokens_seen": 79233024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929187562688064, + "loss": 3.8094, + "theoretical_loss": 4.930810246185626, + "tokens_seen": 79298560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929087261785356, + "loss": 3.5257, + "theoretical_loss": 4.930226971063035, + "tokens_seen": 79364096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928986960882648, + "loss": 3.4985, + "theoretical_loss": 4.929644312123655, + "tokens_seen": 79429632 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492888665997994, + "loss": 3.3258, + "theoretical_loss": 4.929062268208941, + "tokens_seen": 79495168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928786359077232, + "loss": 3.2647, + "theoretical_loss": 4.92848083816348, + "tokens_seen": 79560704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928686058174523, + "loss": 3.8228, + "theoretical_loss": 4.927900020834979, + "tokens_seen": 79626240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928585757271815, + "loss": 3.3943, + "theoretical_loss": 4.927319815074252, + "tokens_seen": 79691776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928485456369107, + "loss": 3.4187, + "theoretical_loss": 4.926740219735214, + "tokens_seen": 79757312 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049283851554664, + "loss": 3.6324, + "theoretical_loss": 4.926161233674863, + "tokens_seen": 79822848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928284854563691, + "loss": 3.627, + "theoretical_loss": 4.925582855753279, + "tokens_seen": 79888384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928184553660984, + "loss": 3.4882, + "theoretical_loss": 4.925005084833603, + "tokens_seen": 79953920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928084252758274, + "loss": 3.8262, + "theoretical_loss": 4.924427919782033, + "tokens_seen": 80019456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927983951855567, + "loss": 3.5192, + "theoretical_loss": 4.923851359467811, + "tokens_seen": 80084992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927883650952859, + "loss": 3.714, + "theoretical_loss": 4.923275402763213, + "tokens_seen": 80150528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927783350050151, + "loss": 3.4187, + "theoretical_loss": 4.922700048543536, + "tokens_seen": 80216064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 56090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1410176753997803, + "objective/train/theoretical_loss": 4.922125295687091, + "objective/train/tokens_used": 100741600, + "theoretical_loss": 4.922125295687091, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927683049147443, + "loss": 3.5733, + "theoretical_loss": 4.922125295687091, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927582748244734, + "loss": 3.396, + "theoretical_loss": 4.92155114307519, + "tokens_seen": 80347136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927482447342026, + "loss": 3.6166, + "theoretical_loss": 4.9209775895921375, + "tokens_seen": 80412672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927382146439318, + "loss": 3.4859, + "theoretical_loss": 4.920404634125217, + "tokens_seen": 80478208 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492728184553661, + "loss": 3.2846, + "theoretical_loss": 4.919832275564684, + "tokens_seen": 80543744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927181544633902, + "loss": 3.653, + "theoretical_loss": 4.9192605128037545, + "tokens_seen": 80609280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927081243731193, + "loss": 3.5658, + "theoretical_loss": 4.918689344738594, + "tokens_seen": 80674816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926980942828485, + "loss": 3.5587, + "theoretical_loss": 4.918118770268305, + "tokens_seen": 80740352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926880641925777, + "loss": 3.3881, + "theoretical_loss": 4.917548788294924, + "tokens_seen": 80805888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926780341023069, + "loss": 3.4548, + "theoretical_loss": 4.916979397723403, + "tokens_seen": 80871424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926680040120361, + "loss": 3.6868, + "theoretical_loss": 4.916410597461606, + "tokens_seen": 80936960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926579739217654, + "loss": 3.6518, + "theoretical_loss": 4.9158423864202945, + "tokens_seen": 81002496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926479438314944, + "loss": 3.4168, + "theoretical_loss": 4.915274763513119, + "tokens_seen": 81068032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926379137412237, + "loss": 3.353, + "theoretical_loss": 4.914707727656609, + "tokens_seen": 81133568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926278836509528, + "loss": 3.3831, + "theoretical_loss": 4.914141277770164, + "tokens_seen": 81199104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926178535606821, + "loss": 3.1851, + "theoretical_loss": 4.913575412776042, + "tokens_seen": 81264640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926078234704113, + "loss": 3.7187, + "theoretical_loss": 4.91301013159935, + "tokens_seen": 81330176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925977933801404, + "loss": 3.5052, + "theoretical_loss": 4.912445433168036, + "tokens_seen": 81395712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925877632898696, + "loss": 3.5131, + "theoretical_loss": 4.911881316412876, + "tokens_seen": 81461248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925777331995988, + "loss": 3.7694, + "theoretical_loss": 4.911317780267467, + "tokens_seen": 81526784 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492567703109328, + "loss": 3.3832, + "theoretical_loss": 4.910754823668215, + "tokens_seen": 81592320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925576730190572, + "loss": 3.8546, + "theoretical_loss": 4.910192445554329, + "tokens_seen": 81657856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925476429287864, + "loss": 3.6127, + "theoretical_loss": 4.909630644867807, + "tokens_seen": 81723392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925376128385155, + "loss": 3.3545, + "theoretical_loss": 4.909069420553429, + "tokens_seen": 81788928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925275827482447, + "loss": 3.5589, + "theoretical_loss": 4.908508771558748, + "tokens_seen": 81854464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 57069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6827940940856934, + "objective/train/theoretical_loss": 4.907948696834077, + "objective/train/tokens_used": 102380000, + "theoretical_loss": 4.907948696834077, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925175526579739, + "loss": 3.7195, + "theoretical_loss": 4.907948696834077, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925075225677031, + "loss": 3.5548, + "theoretical_loss": 4.907389195332484, + "tokens_seen": 81985536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924974924774323, + "loss": 3.807, + "theoretical_loss": 4.906830266009781, + "tokens_seen": 82051072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924874623871615, + "loss": 3.4511, + "theoretical_loss": 4.90627190782451, + "tokens_seen": 82116608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924774322968907, + "loss": 3.7151, + "theoretical_loss": 4.905714119737945, + "tokens_seen": 82182144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924674022066199, + "loss": 3.5237, + "theoretical_loss": 4.9051569007140685, + "tokens_seen": 82247680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924573721163491, + "loss": 3.332, + "theoretical_loss": 4.904600249719573, + "tokens_seen": 82313216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924473420260783, + "loss": 3.504, + "theoretical_loss": 4.90404416572385, + "tokens_seen": 82378752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924373119358075, + "loss": 3.7408, + "theoretical_loss": 4.903488647698972, + "tokens_seen": 82444288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924272818455366, + "loss": 3.334, + "theoretical_loss": 4.902933694619698, + "tokens_seen": 82509824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924172517552658, + "loss": 3.2703, + "theoretical_loss": 4.902379305463451, + "tokens_seen": 82575360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492407221664995, + "loss": 3.5645, + "theoretical_loss": 4.901825479210317, + "tokens_seen": 82640896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923971915747242, + "loss": 3.3278, + "theoretical_loss": 4.9012722148430345, + "tokens_seen": 82706432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923871614844534, + "loss": 3.6277, + "theoretical_loss": 4.900719511346983, + "tokens_seen": 82771968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923771313941825, + "loss": 3.6241, + "theoretical_loss": 4.900167367710176, + "tokens_seen": 82837504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923671013039117, + "loss": 3.5454, + "theoretical_loss": 4.899615782923251, + "tokens_seen": 82903040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923570712136409, + "loss": 3.6864, + "theoretical_loss": 4.899064755979464, + "tokens_seen": 82968576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923470411233702, + "loss": 3.3855, + "theoretical_loss": 4.898514285874674, + "tokens_seen": 83034112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923370110330993, + "loss": 3.4924, + "theoretical_loss": 4.897964371607341, + "tokens_seen": 83099648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923269809428286, + "loss": 3.4934, + "theoretical_loss": 4.897415012178515, + "tokens_seen": 83165184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923169508525576, + "loss": 3.3992, + "theoretical_loss": 4.896866206591823, + "tokens_seen": 83230720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923069207622869, + "loss": 3.5205, + "theoretical_loss": 4.896317953853467, + "tokens_seen": 83296256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922968906720161, + "loss": 3.624, + "theoretical_loss": 4.8957702529722145, + "tokens_seen": 83361792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922868605817453, + "loss": 3.5903, + "theoretical_loss": 4.895223102959381, + "tokens_seen": 83427328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922768304914745, + "loss": 3.7871, + "theoretical_loss": 4.894676502828836, + "tokens_seen": 83492864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 57750, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.804002046585083, + "objective/train/theoretical_loss": 4.894130451596979, + "objective/train/tokens_used": 104018400, + "theoretical_loss": 4.894130451596979, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922668004012036, + "loss": 3.4882, + "theoretical_loss": 4.894130451596979, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922567703109328, + "loss": 3.2586, + "theoretical_loss": 4.893584948282745, + "tokens_seen": 83623936 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492246740220662, + "loss": 3.5408, + "theoretical_loss": 4.893039991907587, + "tokens_seen": 83689472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922367101303912, + "loss": 3.6106, + "theoretical_loss": 4.892495581495468, + "tokens_seen": 83755008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922266800401204, + "loss": 3.2746, + "theoretical_loss": 4.8919517160728585, + "tokens_seen": 83820544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922166499498495, + "loss": 3.5637, + "theoretical_loss": 4.891408394668723, + "tokens_seen": 83886080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922066198595787, + "loss": 3.5945, + "theoretical_loss": 4.89086561631451, + "tokens_seen": 83951616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921965897693079, + "loss": 3.6297, + "theoretical_loss": 4.890323380044153, + "tokens_seen": 84017152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921865596790371, + "loss": 3.6036, + "theoretical_loss": 4.889781684894051, + "tokens_seen": 84082688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921765295887663, + "loss": 3.7009, + "theoretical_loss": 4.889240529903066, + "tokens_seen": 84148224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921664994984956, + "loss": 3.4759, + "theoretical_loss": 4.888699914112516, + "tokens_seen": 84213760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921564694082246, + "loss": 3.4833, + "theoretical_loss": 4.888159836566162, + "tokens_seen": 84279296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921464393179539, + "loss": 3.7335, + "theoretical_loss": 4.887620296310205, + "tokens_seen": 84344832 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492136409227683, + "loss": 3.8101, + "theoretical_loss": 4.887081292393276, + "tokens_seen": 84410368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921263791374123, + "loss": 3.6913, + "theoretical_loss": 4.886542823866424, + "tokens_seen": 84475904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921163490471415, + "loss": 3.5273, + "theoretical_loss": 4.886004889783116, + "tokens_seen": 84541440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921063189568706, + "loss": 3.5718, + "theoretical_loss": 4.885467489199222, + "tokens_seen": 84606976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920962888665998, + "loss": 3.6503, + "theoretical_loss": 4.884930621173009, + "tokens_seen": 84672512 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492086258776329, + "loss": 3.5557, + "theoretical_loss": 4.884394284765135, + "tokens_seen": 84738048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920762286860582, + "loss": 3.5114, + "theoretical_loss": 4.88385847903864, + "tokens_seen": 84803584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920661985957874, + "loss": 3.8069, + "theoretical_loss": 4.883323203058936, + "tokens_seen": 84869120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920561685055166, + "loss": 3.529, + "theoretical_loss": 4.882788455893802, + "tokens_seen": 84934656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920461384152457, + "loss": 3.5584, + "theoretical_loss": 4.882254236613376, + "tokens_seen": 85000192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920361083249749, + "loss": 3.7302, + "theoretical_loss": 4.881720544290144, + "tokens_seen": 85065728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920260782347041, + "loss": 3.4352, + "theoretical_loss": 4.881187377998938, + "tokens_seen": 85131264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 58904, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6649792194366455, + "objective/train/theoretical_loss": 4.880654736816922, + "objective/train/tokens_used": 105656800, + "theoretical_loss": 4.880654736816922, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920160481444333, + "loss": 3.7428, + "theoretical_loss": 4.880654736816922, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920060180541625, + "loss": 3.3637, + "theoretical_loss": 4.880122619823589, + "tokens_seen": 85262336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919959879638916, + "loss": 3.4179, + "theoretical_loss": 4.87959102610075, + "tokens_seen": 85327872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919859578736209, + "loss": 3.3888, + "theoretical_loss": 4.879059954732529, + "tokens_seen": 85393408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049197592778335, + "loss": 3.7427, + "theoretical_loss": 4.878529404805356, + "tokens_seen": 85458944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919658976930793, + "loss": 3.7709, + "theoretical_loss": 4.8779993754079545, + "tokens_seen": 85524480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919558676028084, + "loss": 3.5494, + "theoretical_loss": 4.877469865631338, + "tokens_seen": 85590016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919458375125377, + "loss": 3.5295, + "theoretical_loss": 4.876940874568804, + "tokens_seen": 85655552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919358074222668, + "loss": 3.5878, + "theoretical_loss": 4.876412401315923, + "tokens_seen": 85721088 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491925777331996, + "loss": 3.57, + "theoretical_loss": 4.8758844449705325, + "tokens_seen": 85786624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919157472417252, + "loss": 3.5246, + "theoretical_loss": 4.875357004632728, + "tokens_seen": 85852160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919057171514544, + "loss": 3.3963, + "theoretical_loss": 4.87483007940486, + "tokens_seen": 85917696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918956870611836, + "loss": 3.6586, + "theoretical_loss": 4.874303668391521, + "tokens_seen": 85983232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918856569709127, + "loss": 3.6073, + "theoretical_loss": 4.873777770699542, + "tokens_seen": 86048768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918756268806419, + "loss": 3.5692, + "theoretical_loss": 4.873252385437985, + "tokens_seen": 86114304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918655967903711, + "loss": 3.5077, + "theoretical_loss": 4.872727511718134, + "tokens_seen": 86179840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918555667001003, + "loss": 3.6584, + "theoretical_loss": 4.8722031486534885, + "tokens_seen": 86245376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918455366098295, + "loss": 3.4986, + "theoretical_loss": 4.8716792953597565, + "tokens_seen": 86310912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918355065195586, + "loss": 3.7466, + "theoretical_loss": 4.871155950954847, + "tokens_seen": 86376448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918254764292878, + "loss": 3.627, + "theoretical_loss": 4.870633114558865, + "tokens_seen": 86441984 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491815446339017, + "loss": 3.5282, + "theoretical_loss": 4.8701107852941, + "tokens_seen": 86507520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918054162487463, + "loss": 3.1841, + "theoretical_loss": 4.869588962285023, + "tokens_seen": 86573056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917953861584754, + "loss": 3.6871, + "theoretical_loss": 4.869067644658276, + "tokens_seen": 86638592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917853560682047, + "loss": 3.6656, + "theoretical_loss": 4.86854683154267, + "tokens_seen": 86704128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917753259779337, + "loss": 3.5156, + "theoretical_loss": 4.868026522069172, + "tokens_seen": 86769664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 59520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.652179718017578, + "objective/train/theoretical_loss": 4.867506715370903, + "objective/train/tokens_used": 107295200, + "theoretical_loss": 4.867506715370903, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491765295887663, + "loss": 3.4928, + "theoretical_loss": 4.867506715370903, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917552657973922, + "loss": 3.4802, + "theoretical_loss": 4.866987410583125, + "tokens_seen": 86900736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917452357071214, + "loss": 3.5752, + "theoretical_loss": 4.8664686068432434, + "tokens_seen": 86966272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917352056168506, + "loss": 3.5258, + "theoretical_loss": 4.8659503032907905, + "tokens_seen": 87031808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917251755265797, + "loss": 3.7263, + "theoretical_loss": 4.865432499067425, + "tokens_seen": 87097344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917151454363089, + "loss": 3.7067, + "theoretical_loss": 4.864915193316922, + "tokens_seen": 87162880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917051153460381, + "loss": 3.4247, + "theoretical_loss": 4.864398385185169, + "tokens_seen": 87228416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916950852557673, + "loss": 3.5719, + "theoretical_loss": 4.863882073820154, + "tokens_seen": 87293952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916850551654965, + "loss": 3.5349, + "theoretical_loss": 4.863366258371963, + "tokens_seen": 87359488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916750250752258, + "loss": 3.5815, + "theoretical_loss": 4.862850937992777, + "tokens_seen": 87425024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916649949849548, + "loss": 3.5233, + "theoretical_loss": 4.862336111836852, + "tokens_seen": 87490560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916549648946841, + "loss": 3.5181, + "theoretical_loss": 4.861821779060529, + "tokens_seen": 87556096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916449348044132, + "loss": 3.369, + "theoretical_loss": 4.861307938822215, + "tokens_seen": 87621632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916349047141425, + "loss": 3.3532, + "theoretical_loss": 4.86079459028238, + "tokens_seen": 87687168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916248746238717, + "loss": 3.4634, + "theoretical_loss": 4.860281732603555, + "tokens_seen": 87752704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916148445336008, + "loss": 3.64, + "theoretical_loss": 4.859769364950319, + "tokens_seen": 87818240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049160481444333, + "loss": 3.5164, + "theoretical_loss": 4.859257486489293, + "tokens_seen": 87883776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915947843530592, + "loss": 3.5639, + "theoretical_loss": 4.858746096389139, + "tokens_seen": 87949312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915847542627884, + "loss": 3.4632, + "theoretical_loss": 4.8582351938205495, + "tokens_seen": 88014848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915747241725176, + "loss": 3.2569, + "theoretical_loss": 4.857724777956237, + "tokens_seen": 88080384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915646940822468, + "loss": 3.4019, + "theoretical_loss": 4.857214847970939, + "tokens_seen": 88145920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915546639919759, + "loss": 3.266, + "theoretical_loss": 4.856705403041397, + "tokens_seen": 88211456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915446339017051, + "loss": 3.7935, + "theoretical_loss": 4.8561964423463655, + "tokens_seen": 88276992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915346038114343, + "loss": 3.7385, + "theoretical_loss": 4.8556879650665925, + "tokens_seen": 88342528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915245737211635, + "loss": 3.3908, + "theoretical_loss": 4.855179970384819, + "tokens_seen": 88408064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 60543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.110377788543701, + "objective/train/theoretical_loss": 4.8546724574857745, + "objective/train/tokens_used": 108933600, + "theoretical_loss": 4.8546724574857745, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915145436308927, + "loss": 3.4761, + "theoretical_loss": 4.8546724574857745, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915045135406218, + "loss": 3.442, + "theoretical_loss": 4.854165425556166, + "tokens_seen": 88539136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914944834503511, + "loss": 3.613, + "theoretical_loss": 4.853658873784675, + "tokens_seen": 88604672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914844533600802, + "loss": 3.7501, + "theoretical_loss": 4.853152801361952, + "tokens_seen": 88670208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914744232698095, + "loss": 3.4899, + "theoretical_loss": 4.852647207480604, + "tokens_seen": 88735744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914643931795386, + "loss": 3.6797, + "theoretical_loss": 4.8521420913352, + "tokens_seen": 88801280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914543630892679, + "loss": 3.2736, + "theoretical_loss": 4.851637452122251, + "tokens_seen": 88866816 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491444332998997, + "loss": 3.5697, + "theoretical_loss": 4.851133289040214, + "tokens_seen": 88932352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343029087262, + "loss": 3.6628, + "theoretical_loss": 4.850629601289482, + "tokens_seen": 88997888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914242728184554, + "loss": 3.4603, + "theoretical_loss": 4.850126388072379, + "tokens_seen": 89063424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914142427281846, + "loss": 3.5286, + "theoretical_loss": 4.849623648593151, + "tokens_seen": 89128960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914042126379138, + "loss": 3.4864, + "theoretical_loss": 4.849121382057964, + "tokens_seen": 89194496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913941825476429, + "loss": 3.5269, + "theoretical_loss": 4.848619587674898, + "tokens_seen": 89260032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913841524573721, + "loss": 3.4857, + "theoretical_loss": 4.8481182646539365, + "tokens_seen": 89325568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913741223671013, + "loss": 3.6437, + "theoretical_loss": 4.847617412206963, + "tokens_seen": 89391104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913640922768305, + "loss": 3.4963, + "theoretical_loss": 4.847117029547758, + "tokens_seen": 89456640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913540621865597, + "loss": 3.5452, + "theoretical_loss": 4.846617115891988, + "tokens_seen": 89522176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913440320962888, + "loss": 3.2749, + "theoretical_loss": 4.8461176704572, + "tokens_seen": 89587712 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491334002006018, + "loss": 3.319, + "theoretical_loss": 4.845618692462826, + "tokens_seen": 89653248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913239719157472, + "loss": 3.4855, + "theoretical_loss": 4.845120181130158, + "tokens_seen": 89718784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913139418254765, + "loss": 3.7812, + "theoretical_loss": 4.844622135682359, + "tokens_seen": 89784320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913039117352056, + "loss": 3.1217, + "theoretical_loss": 4.84412455534445, + "tokens_seen": 89849856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912938816449349, + "loss": 3.3833, + "theoretical_loss": 4.843627439343306, + "tokens_seen": 89915392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912838515546639, + "loss": 3.5363, + "theoretical_loss": 4.843130786907647, + "tokens_seen": 89980928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912738214643932, + "loss": 3.4395, + "theoretical_loss": 4.842634597268036, + "tokens_seen": 90046464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 60961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3933932781219482, + "objective/train/theoretical_loss": 4.842138869656871, + "objective/train/tokens_used": 110572000, + "theoretical_loss": 4.842138869656871, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912637913741224, + "loss": 3.5361, + "theoretical_loss": 4.842138869656871, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912537612838516, + "loss": 3.1717, + "theoretical_loss": 4.841643603308384, + "tokens_seen": 90177536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912437311935808, + "loss": 3.6274, + "theoretical_loss": 4.841148797458627, + "tokens_seen": 90243072 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049123370110331, + "loss": 3.5958, + "theoretical_loss": 4.840654451345472, + "tokens_seen": 90308608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912236710130391, + "loss": 3.6702, + "theoretical_loss": 4.840160564208606, + "tokens_seen": 90374144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912136409227683, + "loss": 3.4133, + "theoretical_loss": 4.839667135289522, + "tokens_seen": 90439680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912036108324975, + "loss": 3.3868, + "theoretical_loss": 4.839174163831515, + "tokens_seen": 90505216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911935807422267, + "loss": 3.4971, + "theoretical_loss": 4.838681649079677, + "tokens_seen": 90570752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911835506519559, + "loss": 3.581, + "theoretical_loss": 4.838189590280891, + "tokens_seen": 90636288 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491173520561685, + "loss": 3.5547, + "theoretical_loss": 4.837697986683825, + "tokens_seen": 90701824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911634904714142, + "loss": 3.6512, + "theoretical_loss": 4.837206837538925, + "tokens_seen": 90767360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911534603811434, + "loss": 3.189, + "theoretical_loss": 4.836716142098417, + "tokens_seen": 90832896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911434302908726, + "loss": 3.4003, + "theoretical_loss": 4.836225899616288, + "tokens_seen": 90898432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911334002006019, + "loss": 3.5343, + "theoretical_loss": 4.835736109348293, + "tokens_seen": 90963968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911233701103309, + "loss": 3.491, + "theoretical_loss": 4.8352467705519455, + "tokens_seen": 91029504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911133400200602, + "loss": 3.543, + "theoretical_loss": 4.83475788248651, + "tokens_seen": 91095040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911033099297893, + "loss": 3.5829, + "theoretical_loss": 4.834269444412996, + "tokens_seen": 91160576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910932798395186, + "loss": 3.5038, + "theoretical_loss": 4.83378145559416, + "tokens_seen": 91226112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910832497492478, + "loss": 3.6003, + "theoretical_loss": 4.83329391529449, + "tokens_seen": 91291648 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491073219658977, + "loss": 3.5657, + "theoretical_loss": 4.832806822780208, + "tokens_seen": 91357184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910631895687061, + "loss": 3.3872, + "theoretical_loss": 4.832320177319261, + "tokens_seen": 91422720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910531594784353, + "loss": 3.4167, + "theoretical_loss": 4.831833978181314, + "tokens_seen": 91488256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910431293881645, + "loss": 3.2479, + "theoretical_loss": 4.831348224637752, + "tokens_seen": 91553792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910330992978937, + "loss": 3.5555, + "theoretical_loss": 4.830862915961665, + "tokens_seen": 91619328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910230692076229, + "loss": 3.4961, + "theoretical_loss": 4.830378051427852, + "tokens_seen": 91684864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 62148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0938072204589844, + "objective/train/theoretical_loss": 4.829893630312808, + "objective/train/tokens_used": 112210400, + "theoretical_loss": 4.829893630312808, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491013039117352, + "loss": 3.1482, + "theoretical_loss": 4.829893630312808, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910030090270812, + "loss": 3.4885, + "theoretical_loss": 4.829409651894725, + "tokens_seen": 91815936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909929789368104, + "loss": 3.5385, + "theoretical_loss": 4.8289261154534815, + "tokens_seen": 91881472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909829488465397, + "loss": 3.2172, + "theoretical_loss": 4.828443020270642, + "tokens_seen": 91947008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909729187562688, + "loss": 3.351, + "theoretical_loss": 4.827960365629449, + "tokens_seen": 92012544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909628886659981, + "loss": 3.6262, + "theoretical_loss": 4.827478150814821, + "tokens_seen": 92078080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909528585757272, + "loss": 3.4536, + "theoretical_loss": 4.826996375113339, + "tokens_seen": 92143616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909428284854564, + "loss": 3.4569, + "theoretical_loss": 4.826515037813255, + "tokens_seen": 92209152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909327983951856, + "loss": 3.7922, + "theoretical_loss": 4.826034138204474, + "tokens_seen": 92274688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909227683049148, + "loss": 3.4368, + "theoretical_loss": 4.825553675578557, + "tokens_seen": 92340224 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490912738214644, + "loss": 3.3922, + "theoretical_loss": 4.825073649228713, + "tokens_seen": 92405760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909027081243731, + "loss": 3.5078, + "theoretical_loss": 4.824594058449794, + "tokens_seen": 92471296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908926780341023, + "loss": 3.3106, + "theoretical_loss": 4.824114902538292, + "tokens_seen": 92536832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908826479438315, + "loss": 3.136, + "theoretical_loss": 4.823636180792329, + "tokens_seen": 92602368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908726178535607, + "loss": 3.3286, + "theoretical_loss": 4.823157892511661, + "tokens_seen": 92667904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908625877632899, + "loss": 3.6266, + "theoretical_loss": 4.82268003699766, + "tokens_seen": 92733440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490852557673019, + "loss": 3.5695, + "theoretical_loss": 4.8222026135533245, + "tokens_seen": 92798976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908425275827482, + "loss": 3.487, + "theoretical_loss": 4.821725621483262, + "tokens_seen": 92864512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908324974924774, + "loss": 3.503, + "theoretical_loss": 4.821249060093692, + "tokens_seen": 92930048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908224674022067, + "loss": 3.3758, + "theoretical_loss": 4.820772928692433, + "tokens_seen": 92995584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908124373119358, + "loss": 3.4497, + "theoretical_loss": 4.820297226588908, + "tokens_seen": 93061120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908024072216651, + "loss": 3.6942, + "theoretical_loss": 4.819821953094133, + "tokens_seen": 93126656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907923771313941, + "loss": 3.6533, + "theoretical_loss": 4.819347107520713, + "tokens_seen": 93192192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907823470411234, + "loss": 3.3973, + "theoretical_loss": 4.818872689182836, + "tokens_seen": 93257728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907723169508526, + "loss": 3.2205, + "theoretical_loss": 4.818398697396274, + "tokens_seen": 93323264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 62815, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5069005489349365, + "objective/train/theoretical_loss": 4.817925131478374, + "objective/train/tokens_used": 113848800, + "theoretical_loss": 4.817925131478374, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907622868605818, + "loss": 3.5578, + "theoretical_loss": 4.817925131478374, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490752256770311, + "loss": 3.5471, + "theoretical_loss": 4.817451990748047, + "tokens_seen": 93454336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907422266800401, + "loss": 3.6923, + "theoretical_loss": 4.81697927452578, + "tokens_seen": 93519872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907321965897693, + "loss": 3.5227, + "theoretical_loss": 4.816506982133613, + "tokens_seen": 93585408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907221664994985, + "loss": 3.2295, + "theoretical_loss": 4.816035112895147, + "tokens_seen": 93650944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907121364092277, + "loss": 3.1312, + "theoretical_loss": 4.815563666135532, + "tokens_seen": 93716480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907021063189569, + "loss": 3.6798, + "theoretical_loss": 4.815092641181467, + "tokens_seen": 93782016 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490692076228686, + "loss": 3.7669, + "theoretical_loss": 4.814622037361195, + "tokens_seen": 93847552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906820461384152, + "loss": 3.6417, + "theoretical_loss": 4.814151854004493, + "tokens_seen": 93913088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906720160481444, + "loss": 3.2937, + "theoretical_loss": 4.813682090442674, + "tokens_seen": 93978624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906619859578736, + "loss": 3.2833, + "theoretical_loss": 4.81321274600858, + "tokens_seen": 94044160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906519558676028, + "loss": 3.2788, + "theoretical_loss": 4.812743820036577, + "tokens_seen": 94109696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906419257773321, + "loss": 3.5618, + "theoretical_loss": 4.81227531186255, + "tokens_seen": 94175232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906318956870611, + "loss": 3.5329, + "theoretical_loss": 4.811807220823899, + "tokens_seen": 94240768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906218655967904, + "loss": 3.6156, + "theoretical_loss": 4.811339546259536, + "tokens_seen": 94306304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906118355065195, + "loss": 3.3777, + "theoretical_loss": 4.810872287509877, + "tokens_seen": 94371840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906018054162488, + "loss": 3.5883, + "theoretical_loss": 4.810405443916842, + "tokens_seen": 94437376 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490591775325978, + "loss": 3.6907, + "theoretical_loss": 4.809939014823849, + "tokens_seen": 94502912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905817452357072, + "loss": 3.6602, + "theoretical_loss": 4.809472999575805, + "tokens_seen": 94568448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905717151454363, + "loss": 3.6322, + "theoretical_loss": 4.809007397519108, + "tokens_seen": 94633984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905616850551655, + "loss": 3.4204, + "theoretical_loss": 4.80854220800164, + "tokens_seen": 94699520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905516549648947, + "loss": 3.5405, + "theoretical_loss": 4.808077430372762, + "tokens_seen": 94765056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905416248746239, + "loss": 3.4647, + "theoretical_loss": 4.807613063983311, + "tokens_seen": 94830592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905315947843531, + "loss": 3.7424, + "theoretical_loss": 4.807149108185593, + "tokens_seen": 94896128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905215646940822, + "loss": 3.3372, + "theoretical_loss": 4.806685562333383, + "tokens_seen": 94961664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 64162, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0013229846954346, + "objective/train/theoretical_loss": 4.806222425781916, + "objective/train/tokens_used": 115487200, + "theoretical_loss": 4.806222425781916, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905115346038114, + "loss": 3.4672, + "theoretical_loss": 4.806222425781916, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905015045135406, + "loss": 3.4124, + "theoretical_loss": 4.805759697887886, + "tokens_seen": 95092736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904914744232698, + "loss": 3.5337, + "theoretical_loss": 4.80529737800944, + "tokens_seen": 95158272 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490481444332999, + "loss": 3.1987, + "theoretical_loss": 4.804835465506176, + "tokens_seen": 95223808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904714142427281, + "loss": 3.3264, + "theoretical_loss": 4.8043739597391335, + "tokens_seen": 95289344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904613841524574, + "loss": 3.421, + "theoretical_loss": 4.803912860070795, + "tokens_seen": 95354880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904513540621865, + "loss": 3.5238, + "theoretical_loss": 4.803452165865082, + "tokens_seen": 95420416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904413239719158, + "loss": 3.4263, + "theoretical_loss": 4.802991876487344, + "tokens_seen": 95485952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904312938816449, + "loss": 3.3975, + "theoretical_loss": 4.802531991304359, + "tokens_seen": 95551488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904212637913742, + "loss": 3.5391, + "theoretical_loss": 4.802072509684331, + "tokens_seen": 95617024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904112337011033, + "loss": 3.4524, + "theoretical_loss": 4.801613430996884, + "tokens_seen": 95682560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904012036108325, + "loss": 3.5944, + "theoretical_loss": 4.801154754613056, + "tokens_seen": 95748096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903911735205617, + "loss": 3.2372, + "theoretical_loss": 4.800696479905296, + "tokens_seen": 95813632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903811434302909, + "loss": 3.1815, + "theoretical_loss": 4.800238606247462, + "tokens_seen": 95879168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903711133400201, + "loss": 3.4315, + "theoretical_loss": 4.799781133014815, + "tokens_seen": 95944704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903610832497492, + "loss": 3.2851, + "theoretical_loss": 4.799324059584013, + "tokens_seen": 96010240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903510531594784, + "loss": 3.6757, + "theoretical_loss": 4.798867385333111, + "tokens_seen": 96075776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903410230692076, + "loss": 3.3841, + "theoretical_loss": 4.798411109641553, + "tokens_seen": 96141312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903309929789368, + "loss": 3.6343, + "theoretical_loss": 4.797955231890175, + "tokens_seen": 96206848 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490320962888666, + "loss": 3.3026, + "theoretical_loss": 4.797499751461188, + "tokens_seen": 96272384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903109327983952, + "loss": 3.3258, + "theoretical_loss": 4.797044667738188, + "tokens_seen": 96337920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903009027081243, + "loss": 3.4414, + "theoretical_loss": 4.796589980106143, + "tokens_seen": 96403456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902908726178535, + "loss": 3.2512, + "theoretical_loss": 4.796135687951393, + "tokens_seen": 96468992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902808425275828, + "loss": 3.4072, + "theoretical_loss": 4.795681790661644, + "tokens_seen": 96534528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902708124373119, + "loss": 3.4574, + "theoretical_loss": 4.795228287625964, + "tokens_seen": 96600064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 64775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.715846538543701, + "objective/train/theoretical_loss": 4.794775178234782, + "objective/train/tokens_used": 117125600, + "theoretical_loss": 4.794775178234782, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902607823470412, + "loss": 3.622, + "theoretical_loss": 4.794775178234782, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902507522567703, + "loss": 3.3135, + "theoretical_loss": 4.794322461879881, + "tokens_seen": 96731136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902407221664995, + "loss": 3.3367, + "theoretical_loss": 4.793870137954395, + "tokens_seen": 96796672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902306920762287, + "loss": 3.3923, + "theoretical_loss": 4.793418205852804, + "tokens_seen": 96862208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902206619859579, + "loss": 3.2944, + "theoretical_loss": 4.7929666649709315, + "tokens_seen": 96927744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902106318956871, + "loss": 3.4677, + "theoretical_loss": 4.7925155147059435, + "tokens_seen": 96993280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902006018054163, + "loss": 3.5354, + "theoretical_loss": 4.792064754456337, + "tokens_seen": 97058816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901905717151454, + "loss": 3.4739, + "theoretical_loss": 4.791614383621943, + "tokens_seen": 97124352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901805416248746, + "loss": 3.4712, + "theoretical_loss": 4.79116440160392, + "tokens_seen": 97189888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901705115346038, + "loss": 3.6806, + "theoretical_loss": 4.790714807804751, + "tokens_seen": 97255424 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490160481444333, + "loss": 3.2497, + "theoretical_loss": 4.790265601628235, + "tokens_seen": 97320960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901504513540623, + "loss": 3.3692, + "theoretical_loss": 4.789816782479494, + "tokens_seen": 97386496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901404212637913, + "loss": 3.576, + "theoretical_loss": 4.789368349764956, + "tokens_seen": 97452032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901303911735206, + "loss": 3.4743, + "theoretical_loss": 4.788920302892365, + "tokens_seen": 97517568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901203610832497, + "loss": 3.1291, + "theoretical_loss": 4.7884726412707606, + "tokens_seen": 97583104 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490110330992979, + "loss": 3.5709, + "theoretical_loss": 4.788025364310492, + "tokens_seen": 97648640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901003009027082, + "loss": 3.3566, + "theoretical_loss": 4.787578471423203, + "tokens_seen": 97714176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900902708124374, + "loss": 3.6399, + "theoretical_loss": 4.787131962021827, + "tokens_seen": 97779712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900802407221665, + "loss": 3.577, + "theoretical_loss": 4.786685835520595, + "tokens_seen": 97845248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900702106318957, + "loss": 3.4237, + "theoretical_loss": 4.786240091335019, + "tokens_seen": 97910784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900601805416249, + "loss": 3.3724, + "theoretical_loss": 4.785794728881895, + "tokens_seen": 97976320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900501504513541, + "loss": 3.3342, + "theoretical_loss": 4.785349747579299, + "tokens_seen": 98041856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900401203610833, + "loss": 3.5862, + "theoretical_loss": 4.784905146846581, + "tokens_seen": 98107392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900300902708124, + "loss": 3.3465, + "theoretical_loss": 4.784460926104362, + "tokens_seen": 98172928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900200601805416, + "loss": 3.3709, + "theoretical_loss": 4.7840170847745345, + "tokens_seen": 98238464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 65599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.469925880432129, + "objective/train/theoretical_loss": 4.783573622280253, + "objective/train/tokens_used": 118764000, + "theoretical_loss": 4.783573622280253, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900100300902708, + "loss": 3.5123, + "theoretical_loss": 4.783573622280253, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049, + "loss": 3.1796, + "theoretical_loss": 4.7831305380459295, + "tokens_seen": 98369536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899899699097292, + "loss": 3.5071, + "theoretical_loss": 4.78268783149724, + "tokens_seen": 98435072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899799398194583, + "loss": 3.1618, + "theoretical_loss": 4.782245502061111, + "tokens_seen": 98500608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899699097291876, + "loss": 3.5861, + "theoretical_loss": 4.781803549165718, + "tokens_seen": 98566144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899598796389167, + "loss": 3.7046, + "theoretical_loss": 4.781361972240484, + "tokens_seen": 98631680 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489949849548646, + "loss": 3.4213, + "theoretical_loss": 4.780920770716076, + "tokens_seen": 98697216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899398194583751, + "loss": 3.5335, + "theoretical_loss": 4.780479944024398, + "tokens_seen": 98762752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899297893681044, + "loss": 3.3783, + "theoretical_loss": 4.7800394915985915, + "tokens_seen": 98828288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899197592778335, + "loss": 3.5404, + "theoretical_loss": 4.779599412873031, + "tokens_seen": 98893824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899097291875627, + "loss": 3.4117, + "theoretical_loss": 4.77915970728332, + "tokens_seen": 98959360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898996990972919, + "loss": 3.1311, + "theoretical_loss": 4.7787203742662845, + "tokens_seen": 99024896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898896690070211, + "loss": 3.3778, + "theoretical_loss": 4.778281413259977, + "tokens_seen": 99090432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898796389167503, + "loss": 3.3097, + "theoretical_loss": 4.777842823703665, + "tokens_seen": 99155968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898696088264794, + "loss": 3.4696, + "theoretical_loss": 4.777404605037833, + "tokens_seen": 99221504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898595787362086, + "loss": 3.5204, + "theoretical_loss": 4.7769667567041765, + "tokens_seen": 99287040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898495486459378, + "loss": 3.5602, + "theoretical_loss": 4.776529278145597, + "tokens_seen": 99352576 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489839518555667, + "loss": 3.4991, + "theoretical_loss": 4.776092168806207, + "tokens_seen": 99418112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898294884653962, + "loss": 3.6413, + "theoretical_loss": 4.775655428131313, + "tokens_seen": 99483648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898194583751254, + "loss": 3.6154, + "theoretical_loss": 4.775219055567424, + "tokens_seen": 99549184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898094282848545, + "loss": 3.4266, + "theoretical_loss": 4.7747830505622435, + "tokens_seen": 99614720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897993981945837, + "loss": 3.2347, + "theoretical_loss": 4.774347412564666, + "tokens_seen": 99680256 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489789368104313, + "loss": 3.5374, + "theoretical_loss": 4.773912141024771, + "tokens_seen": 99745792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897793380140421, + "loss": 3.3582, + "theoretical_loss": 4.773477235393826, + "tokens_seen": 99811328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897693079237714, + "loss": 3.4578, + "theoretical_loss": 4.773042695124279, + "tokens_seen": 99876864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 66410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.062344551086426, + "objective/train/theoretical_loss": 4.772608519669756, + "objective/train/tokens_used": 120402400, + "theoretical_loss": 4.772608519669756, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897592778335005, + "loss": 3.5402, + "theoretical_loss": 4.772608519669756, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897492477432297, + "loss": 3.5992, + "theoretical_loss": 4.772174708485057, + "tokens_seen": 100007936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897392176529589, + "loss": 3.4125, + "theoretical_loss": 4.771741261026155, + "tokens_seen": 100073472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897291875626881, + "loss": 3.3972, + "theoretical_loss": 4.771308176750187, + "tokens_seen": 100139008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897191574724173, + "loss": 3.7219, + "theoretical_loss": 4.77087545511546, + "tokens_seen": 100204544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897091273821465, + "loss": 3.5976, + "theoretical_loss": 4.77044309558144, + "tokens_seen": 100270080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896990972918756, + "loss": 3.5781, + "theoretical_loss": 4.7700110976087515, + "tokens_seen": 100335616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896890672016048, + "loss": 3.4885, + "theoretical_loss": 4.769579460659173, + "tokens_seen": 100401152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489679037111334, + "loss": 3.6787, + "theoretical_loss": 4.769148184195636, + "tokens_seen": 100466688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896690070210632, + "loss": 3.5478, + "theoretical_loss": 4.768717267682223, + "tokens_seen": 100532224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896589769307924, + "loss": 3.4238, + "theoretical_loss": 4.768286710584157, + "tokens_seen": 100597760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896489468405215, + "loss": 3.3217, + "theoretical_loss": 4.767856512367805, + "tokens_seen": 100663296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896389167502507, + "loss": 3.3783, + "theoretical_loss": 4.767426672500677, + "tokens_seen": 100728832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896288866599799, + "loss": 3.2931, + "theoretical_loss": 4.766997190451413, + "tokens_seen": 100794368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896188565697091, + "loss": 3.6434, + "theoretical_loss": 4.766568065689789, + "tokens_seen": 100859904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896088264794384, + "loss": 3.5874, + "theoretical_loss": 4.766139297686712, + "tokens_seen": 100925440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895987963891674, + "loss": 3.4533, + "theoretical_loss": 4.76571088591421, + "tokens_seen": 100990976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895887662988967, + "loss": 3.6816, + "theoretical_loss": 4.7652828298454395, + "tokens_seen": 101056512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895787362086259, + "loss": 3.3644, + "theoretical_loss": 4.764855128954675, + "tokens_seen": 101122048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895687061183551, + "loss": 3.2458, + "theoretical_loss": 4.764427782717309, + "tokens_seen": 101187584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895586760280843, + "loss": 3.3975, + "theoretical_loss": 4.764000790609848, + "tokens_seen": 101253120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895486459378135, + "loss": 3.3918, + "theoretical_loss": 4.763574152109907, + "tokens_seen": 101318656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895386158475426, + "loss": 3.4413, + "theoretical_loss": 4.763147866696213, + "tokens_seen": 101384192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895285857572718, + "loss": 3.154, + "theoretical_loss": 4.762721933848593, + "tokens_seen": 101449728 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489518555667001, + "loss": 3.367, + "theoretical_loss": 4.762296353047979, + "tokens_seen": 101515264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 66745, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.258596897125244, + "objective/train/theoretical_loss": 4.7618711237764035, + "objective/train/tokens_used": 122040800, + "theoretical_loss": 4.7618711237764035, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895085255767302, + "loss": 3.3147, + "theoretical_loss": 4.7618711237764035, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894984954864594, + "loss": 3.2477, + "theoretical_loss": 4.761446245516988, + "tokens_seen": 101646336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894884653961885, + "loss": 3.5315, + "theoretical_loss": 4.7610217177539536, + "tokens_seen": 101711872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894784353059178, + "loss": 3.3191, + "theoretical_loss": 4.760597539972606, + "tokens_seen": 101777408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894684052156469, + "loss": 3.0442, + "theoretical_loss": 4.760173711659341, + "tokens_seen": 101842944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894583751253762, + "loss": 3.4036, + "theoretical_loss": 4.759750232301635, + "tokens_seen": 101908480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894483450351053, + "loss": 3.3851, + "theoretical_loss": 4.759327101388048, + "tokens_seen": 101974016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894383149448346, + "loss": 3.5575, + "theoretical_loss": 4.7589043184082165, + "tokens_seen": 102039552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894282848545637, + "loss": 3.3678, + "theoretical_loss": 4.75848188285285, + "tokens_seen": 102105088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894182547642929, + "loss": 3.5807, + "theoretical_loss": 4.7580597942137315, + "tokens_seen": 102170624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894082246740221, + "loss": 3.4755, + "theoretical_loss": 4.757638051983713, + "tokens_seen": 102236160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893981945837513, + "loss": 3.308, + "theoretical_loss": 4.757216655656713, + "tokens_seen": 102301696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893881644934805, + "loss": 3.4312, + "theoretical_loss": 4.756795604727708, + "tokens_seen": 102367232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893781344032096, + "loss": 3.4421, + "theoretical_loss": 4.7563748986927425, + "tokens_seen": 102432768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893681043129388, + "loss": 3.3071, + "theoretical_loss": 4.755954537048913, + "tokens_seen": 102498304 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489358074222668, + "loss": 3.7184, + "theoretical_loss": 4.755534519294368, + "tokens_seen": 102563840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893480441323972, + "loss": 3.6644, + "theoretical_loss": 4.7551148449283165, + "tokens_seen": 102629376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893380140421264, + "loss": 3.3233, + "theoretical_loss": 4.754695513451004, + "tokens_seen": 102694912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893279839518556, + "loss": 3.6128, + "theoretical_loss": 4.754276524363733, + "tokens_seen": 102760448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893179538615847, + "loss": 3.4243, + "theoretical_loss": 4.75385787716884, + "tokens_seen": 102825984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893079237713139, + "loss": 3.6836, + "theoretical_loss": 4.7534395713697055, + "tokens_seen": 102891520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892978936810432, + "loss": 3.6634, + "theoretical_loss": 4.753021606470748, + "tokens_seen": 102957056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892878635907723, + "loss": 3.3886, + "theoretical_loss": 4.752603981977417, + "tokens_seen": 103022592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892778335005016, + "loss": 3.4605, + "theoretical_loss": 4.752186697396198, + "tokens_seen": 103088128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892678034102307, + "loss": 3.3429, + "theoretical_loss": 4.7517697522345985, + "tokens_seen": 103153664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 68050, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.391291379928589, + "objective/train/theoretical_loss": 4.7513531460011595, + "objective/train/tokens_used": 123679200, + "theoretical_loss": 4.7513531460011595, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892577733199599, + "loss": 3.488, + "theoretical_loss": 4.7513531460011595, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892477432296891, + "loss": 3.4412, + "theoretical_loss": 4.750936878205438, + "tokens_seen": 103284736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892377131394183, + "loss": 3.1551, + "theoretical_loss": 4.750520948358019, + "tokens_seen": 103350272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892276830491475, + "loss": 3.3288, + "theoretical_loss": 4.750105355970497, + "tokens_seen": 103415808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892176529588767, + "loss": 3.3249, + "theoretical_loss": 4.749690100555487, + "tokens_seen": 103481344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892076228686058, + "loss": 3.4856, + "theoretical_loss": 4.749275181626612, + "tokens_seen": 103546880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489197592778335, + "loss": 3.447, + "theoretical_loss": 4.748860598698509, + "tokens_seen": 103612416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891875626880642, + "loss": 3.3712, + "theoretical_loss": 4.748446351286817, + "tokens_seen": 103677952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891775325977934, + "loss": 3.3355, + "theoretical_loss": 4.748032438908181, + "tokens_seen": 103743488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891675025075226, + "loss": 3.6823, + "theoretical_loss": 4.747618861080247, + "tokens_seen": 103809024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891574724172517, + "loss": 3.5451, + "theoretical_loss": 4.74720561732166, + "tokens_seen": 103874560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891474423269809, + "loss": 3.3377, + "theoretical_loss": 4.746792707152057, + "tokens_seen": 103940096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891374122367101, + "loss": 3.2533, + "theoretical_loss": 4.746380130092074, + "tokens_seen": 104005632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891273821464393, + "loss": 3.5114, + "theoretical_loss": 4.745967885663333, + "tokens_seen": 104071168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891173520561686, + "loss": 3.5891, + "theoretical_loss": 4.745555973388447, + "tokens_seen": 104136704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891073219658976, + "loss": 3.5146, + "theoretical_loss": 4.745144392791008, + "tokens_seen": 104202240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890972918756269, + "loss": 3.477, + "theoretical_loss": 4.744733143395598, + "tokens_seen": 104267776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890872617853561, + "loss": 3.5138, + "theoretical_loss": 4.744322224727773, + "tokens_seen": 104333312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890772316950853, + "loss": 3.4684, + "theoretical_loss": 4.743911636314069, + "tokens_seen": 104398848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890672016048145, + "loss": 3.3884, + "theoretical_loss": 4.743501377681995, + "tokens_seen": 104464384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890571715145437, + "loss": 3.2946, + "theoretical_loss": 4.743091448360034, + "tokens_seen": 104529920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890471414242728, + "loss": 3.362, + "theoretical_loss": 4.742681847877636, + "tokens_seen": 104595456 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489037111334002, + "loss": 3.1456, + "theoretical_loss": 4.742272575765218, + "tokens_seen": 104660992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890270812437312, + "loss": 3.6056, + "theoretical_loss": 4.741863631554163, + "tokens_seen": 104726528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890170511534604, + "loss": 3.5804, + "theoretical_loss": 4.741455014776813, + "tokens_seen": 104792064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 68417, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.796355724334717, + "objective/train/theoretical_loss": 4.741046724966468, + "objective/train/tokens_used": 125317600, + "theoretical_loss": 4.741046724966468, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890070210631896, + "loss": 3.5708, + "theoretical_loss": 4.741046724966468, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889969909729187, + "loss": 3.1131, + "theoretical_loss": 4.74063876165739, + "tokens_seen": 104923136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889869608826479, + "loss": 3.4213, + "theoretical_loss": 4.740231124384789, + "tokens_seen": 104988672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889769307923771, + "loss": 3.3977, + "theoretical_loss": 4.739823812684829, + "tokens_seen": 105054208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889669007021063, + "loss": 3.345, + "theoretical_loss": 4.739416826094621, + "tokens_seen": 105119744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889568706118355, + "loss": 3.4305, + "theoretical_loss": 4.739010164152224, + "tokens_seen": 105185280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889468405215647, + "loss": 3.393, + "theoretical_loss": 4.7386038263966395, + "tokens_seen": 105250816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889368104312939, + "loss": 3.4701, + "theoretical_loss": 4.73819781236781, + "tokens_seen": 105316352 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488926780341023, + "loss": 3.2626, + "theoretical_loss": 4.737792121606619, + "tokens_seen": 105381888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889167502507523, + "loss": 3.364, + "theoretical_loss": 4.737386753654883, + "tokens_seen": 105447424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889067201604815, + "loss": 3.4215, + "theoretical_loss": 4.736981708055353, + "tokens_seen": 105512960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888966900702107, + "loss": 3.2655, + "theoretical_loss": 4.7365769843517125, + "tokens_seen": 105578496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888866599799398, + "loss": 3.2771, + "theoretical_loss": 4.736172582088571, + "tokens_seen": 105644032 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488876629889669, + "loss": 3.3345, + "theoretical_loss": 4.735768500811469, + "tokens_seen": 105709568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888665997993982, + "loss": 3.4764, + "theoretical_loss": 4.735364740066865, + "tokens_seen": 105775104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888565697091274, + "loss": 3.7094, + "theoretical_loss": 4.734961299402143, + "tokens_seen": 105840640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888465396188566, + "loss": 3.4389, + "theoretical_loss": 4.734558178365605, + "tokens_seen": 105906176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888365095285858, + "loss": 3.5066, + "theoretical_loss": 4.734155376506466, + "tokens_seen": 105971712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888264794383149, + "loss": 3.2004, + "theoretical_loss": 4.733752893374863, + "tokens_seen": 106037248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888164493480441, + "loss": 3.9504, + "theoretical_loss": 4.733350728521835, + "tokens_seen": 106102784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888064192577733, + "loss": 3.3078, + "theoretical_loss": 4.732948881499336, + "tokens_seen": 106168320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887963891675025, + "loss": 3.258, + "theoretical_loss": 4.7325473518602275, + "tokens_seen": 106233856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887863590772317, + "loss": 3.4263, + "theoretical_loss": 4.732146139158271, + "tokens_seen": 106299392 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488776328986961, + "loss": 3.2168, + "theoretical_loss": 4.731745242948134, + "tokens_seen": 106364928 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048876629889669, + "loss": 3.7064, + "theoretical_loss": 4.731344662785381, + "tokens_seen": 106430464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 69666, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1660373210906982, + "objective/train/theoretical_loss": 4.730944398226477, + "objective/train/tokens_used": 126956000, + "theoretical_loss": 4.730944398226477, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887562688064193, + "loss": 3.4624, + "theoretical_loss": 4.730944398226477, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887462387161484, + "loss": 3.2648, + "theoretical_loss": 4.730544448828777, + "tokens_seen": 106561536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887362086258777, + "loss": 3.3377, + "theoretical_loss": 4.730144814150533, + "tokens_seen": 106627072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887261785356069, + "loss": 3.182, + "theoretical_loss": 4.729745493750885, + "tokens_seen": 106692608 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488716148445336, + "loss": 3.5011, + "theoretical_loss": 4.729346487189863, + "tokens_seen": 106758144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887061183550652, + "loss": 3.4765, + "theoretical_loss": 4.728947794028379, + "tokens_seen": 106823680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886960882647944, + "loss": 3.5698, + "theoretical_loss": 4.728549413828231, + "tokens_seen": 106889216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886860581745236, + "loss": 3.5244, + "theoretical_loss": 4.728151346152098, + "tokens_seen": 106954752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886760280842528, + "loss": 3.2724, + "theoretical_loss": 4.727753590563535, + "tokens_seen": 107020288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886659979939819, + "loss": 3.2698, + "theoretical_loss": 4.727356146626976, + "tokens_seen": 107085824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886559679037111, + "loss": 3.3792, + "theoretical_loss": 4.726959013907727, + "tokens_seen": 107151360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886459378134403, + "loss": 3.0486, + "theoretical_loss": 4.7265621919719685, + "tokens_seen": 107216896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886359077231695, + "loss": 3.2127, + "theoretical_loss": 4.726165680386746, + "tokens_seen": 107282432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886258776328988, + "loss": 3.4488, + "theoretical_loss": 4.725769478719978, + "tokens_seen": 107347968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886158475426278, + "loss": 3.6296, + "theoretical_loss": 4.725373586540442, + "tokens_seen": 107413504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886058174523571, + "loss": 3.1497, + "theoretical_loss": 4.724978003417782, + "tokens_seen": 107479040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885957873620863, + "loss": 3.0499, + "theoretical_loss": 4.724582728922501, + "tokens_seen": 107544576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885857572718155, + "loss": 3.3971, + "theoretical_loss": 4.724187762625959, + "tokens_seen": 107610112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885757271815447, + "loss": 3.4614, + "theoretical_loss": 4.723793104100376, + "tokens_seen": 107675648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656970912739, + "loss": 3.2867, + "theoretical_loss": 4.72339875291882, + "tokens_seen": 107741184 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488555667001003, + "loss": 3.5243, + "theoretical_loss": 4.723004708655214, + "tokens_seen": 107806720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885456369107322, + "loss": 3.471, + "theoretical_loss": 4.722610970884331, + "tokens_seen": 107872256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885356068204614, + "loss": 3.4896, + "theoretical_loss": 4.722217539181788, + "tokens_seen": 107937792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885255767301906, + "loss": 3.5163, + "theoretical_loss": 4.7218244131240485, + "tokens_seen": 108003328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885155466399198, + "loss": 3.1836, + "theoretical_loss": 4.721431592288419, + "tokens_seen": 108068864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 70233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.18623423576355, + "objective/train/theoretical_loss": 4.721039076253046, + "objective/train/tokens_used": 128594400, + "theoretical_loss": 4.721039076253046, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488505516549649, + "loss": 3.2707, + "theoretical_loss": 4.721039076253046, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884954864593781, + "loss": 3.3744, + "theoretical_loss": 4.720646864596914, + "tokens_seen": 108199936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884854563691073, + "loss": 3.2403, + "theoretical_loss": 4.720254956899844, + "tokens_seen": 108265472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884754262788365, + "loss": 3.4192, + "theoretical_loss": 4.719863352742491, + "tokens_seen": 108331008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884653961885657, + "loss": 3.2221, + "theoretical_loss": 4.719472051706341, + "tokens_seen": 108396544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884553660982949, + "loss": 3.4325, + "theoretical_loss": 4.71908105337371, + "tokens_seen": 108462080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884453360080241, + "loss": 3.3533, + "theoretical_loss": 4.718690357327743, + "tokens_seen": 108527616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884353059177532, + "loss": 3.5112, + "theoretical_loss": 4.718299963152407, + "tokens_seen": 108593152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884252758274825, + "loss": 3.2922, + "theoretical_loss": 4.7179098704324955, + "tokens_seen": 108658688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884152457372117, + "loss": 3.4903, + "theoretical_loss": 4.717520078753621, + "tokens_seen": 108724224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884052156469409, + "loss": 3.5257, + "theoretical_loss": 4.717130587702217, + "tokens_seen": 108789760 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048839518555667, + "loss": 3.371, + "theoretical_loss": 4.716741396865531, + "tokens_seen": 108855296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883851554663992, + "loss": 3.388, + "theoretical_loss": 4.716352505831628, + "tokens_seen": 108920832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883751253761284, + "loss": 3.5684, + "theoretical_loss": 4.715963914189384, + "tokens_seen": 108986368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883650952858576, + "loss": 3.3996, + "theoretical_loss": 4.715575621528487, + "tokens_seen": 109051904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883550651955868, + "loss": 3.1784, + "theoretical_loss": 4.71518762743943, + "tokens_seen": 109117440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488345035105316, + "loss": 3.2919, + "theoretical_loss": 4.714799931513517, + "tokens_seen": 109182976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883350050150451, + "loss": 3.1558, + "theoretical_loss": 4.714412533342855, + "tokens_seen": 109248512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883249749247743, + "loss": 3.5333, + "theoretical_loss": 4.7140254325203506, + "tokens_seen": 109314048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883149448345035, + "loss": 3.6226, + "theoretical_loss": 4.713638628639713, + "tokens_seen": 109379584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883049147442327, + "loss": 3.2969, + "theoretical_loss": 4.713252121295449, + "tokens_seen": 109445120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882948846539619, + "loss": 3.6223, + "theoretical_loss": 4.712865910082863, + "tokens_seen": 109510656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882848545636911, + "loss": 3.4854, + "theoretical_loss": 4.712479994598052, + "tokens_seen": 109576192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882748244734203, + "loss": 3.4307, + "theoretical_loss": 4.7120943744379025, + "tokens_seen": 109641728 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048826479438314946, + "loss": 3.3526, + "theoretical_loss": 4.7117090492000955, + "tokens_seen": 109707264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 70991, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7265214920043945, + "objective/train/theoretical_loss": 4.711324018483099, + "objective/train/tokens_used": 130232800, + "theoretical_loss": 4.711324018483099, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048825476429287864, + "loss": 3.4221, + "theoretical_loss": 4.711324018483099, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882447342026078, + "loss": 3.156, + "theoretical_loss": 4.710939281886164, + "tokens_seen": 109838336 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488234704112337, + "loss": 3.5109, + "theoretical_loss": 4.71055483900933, + "tokens_seen": 109903872 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822467402206624, + "loss": 3.6234, + "theoretical_loss": 4.710170689453415, + "tokens_seen": 109969408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048821464393179536, + "loss": 3.3112, + "theoretical_loss": 4.709786832820017, + "tokens_seen": 110034944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882046138415246, + "loss": 3.3367, + "theoretical_loss": 4.7094032687115135, + "tokens_seen": 110100480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881945837512537, + "loss": 3.3711, + "theoretical_loss": 4.709019996731058, + "tokens_seen": 110166016 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818455366098296, + "loss": 3.1866, + "theoretical_loss": 4.708637016482579, + "tokens_seen": 110231552 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048817452357071214, + "loss": 3.3009, + "theoretical_loss": 4.708254327570774, + "tokens_seen": 110297088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881644934804413, + "loss": 3.6798, + "theoretical_loss": 4.7078719296011124, + "tokens_seen": 110362624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881544633901705, + "loss": 3.5188, + "theoretical_loss": 4.7074898221798325, + "tokens_seen": 110428160 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048814443329989974, + "loss": 3.1, + "theoretical_loss": 4.707108004913936, + "tokens_seen": 110493696 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048813440320962887, + "loss": 3.5579, + "theoretical_loss": 4.706726477411193, + "tokens_seen": 110559232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881243731193581, + "loss": 3.3834, + "theoretical_loss": 4.706345239280133, + "tokens_seen": 110624768 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048811434302908723, + "loss": 3.3456, + "theoretical_loss": 4.705964290130046, + "tokens_seen": 110690304 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048810431293881646, + "loss": 3.3642, + "theoretical_loss": 4.705583629570983, + "tokens_seen": 110755840 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048809428284854564, + "loss": 3.2147, + "theoretical_loss": 4.705203257213746, + "tokens_seen": 110821376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880842527582748, + "loss": 3.4359, + "theoretical_loss": 4.7048231726699, + "tokens_seen": 110886912 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488074222668004, + "loss": 3.163, + "theoretical_loss": 4.7044433755517545, + "tokens_seen": 110952448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880641925777332, + "loss": 3.2166, + "theoretical_loss": 4.704063865472374, + "tokens_seen": 111017984 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048805416248746237, + "loss": 3.2815, + "theoretical_loss": 4.703684642045571, + "tokens_seen": 111083520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880441323971916, + "loss": 3.3855, + "theoretical_loss": 4.703305704885906, + "tokens_seen": 111149056 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048803410230692073, + "loss": 3.476, + "theoretical_loss": 4.7029270536086845, + "tokens_seen": 111214592 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802407221664997, + "loss": 3.1811, + "theoretical_loss": 4.702548687829953, + "tokens_seen": 111280128 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048801404212637915, + "loss": 3.212, + "theoretical_loss": 4.702170607166503, + "tokens_seen": 111345664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 71566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1423094272613525, + "objective/train/theoretical_loss": 4.701792811235861, + "objective/train/tokens_used": 131871200, + "theoretical_loss": 4.701792811235861, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048800401203610833, + "loss": 3.3106, + "theoretical_loss": 4.701792811235861, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879939819458375, + "loss": 3.3804, + "theoretical_loss": 4.701415299656295, + "tokens_seen": 111476736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879839518555667, + "loss": 3.3137, + "theoretical_loss": 4.701038072046808, + "tokens_seen": 111542272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879739217652959, + "loss": 3.4184, + "theoretical_loss": 4.7006611280271375, + "tokens_seen": 111607808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879638916750251, + "loss": 3.5134, + "theoretical_loss": 4.700284467217749, + "tokens_seen": 111673344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879538615847543, + "loss": 3.515, + "theoretical_loss": 4.699908089239845, + "tokens_seen": 111738880 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048794383149448347, + "loss": 3.4391, + "theoretical_loss": 4.6995319937153495, + "tokens_seen": 111804416 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048793380140421265, + "loss": 3.4928, + "theoretical_loss": 4.699156180266918, + "tokens_seen": 111869952 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048792377131394183, + "loss": 3.4204, + "theoretical_loss": 4.69878064851793, + "tokens_seen": 111935488 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048791374122367107, + "loss": 3.2864, + "theoretical_loss": 4.698405398092486, + "tokens_seen": 112001024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879037111334002, + "loss": 3.52, + "theoretical_loss": 4.698030428615411, + "tokens_seen": 112066560 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048789368104312943, + "loss": 3.2593, + "theoretical_loss": 4.6976557397122445, + "tokens_seen": 112132096 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048788365095285856, + "loss": 3.3313, + "theoretical_loss": 4.697281331009249, + "tokens_seen": 112197632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878736208625878, + "loss": 3.1676, + "theoretical_loss": 4.6969072021333975, + "tokens_seen": 112263168 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048786359077231697, + "loss": 3.4483, + "theoretical_loss": 4.696533352712381, + "tokens_seen": 112328704 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048785356068204615, + "loss": 3.3785, + "theoretical_loss": 4.696159782374602, + "tokens_seen": 112394240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048784353059177533, + "loss": 3.0219, + "theoretical_loss": 4.695786490749171, + "tokens_seen": 112459776 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048783350050150457, + "loss": 3.3436, + "theoretical_loss": 4.6954134774659115, + "tokens_seen": 112525312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878234704112337, + "loss": 3.3403, + "theoretical_loss": 4.695040742155349, + "tokens_seen": 112590848 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048781344032096293, + "loss": 3.4199, + "theoretical_loss": 4.694668284448717, + "tokens_seen": 112656384 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048780341023069206, + "loss": 3.0947, + "theoretical_loss": 4.694296103977953, + "tokens_seen": 112721920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877933801404213, + "loss": 2.9975, + "theoretical_loss": 4.693924200375694, + "tokens_seen": 112787456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877833500501505, + "loss": 3.2263, + "theoretical_loss": 4.693552573275277, + "tokens_seen": 112852992 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048777331995987966, + "loss": 3.3911, + "theoretical_loss": 4.69318122231074, + "tokens_seen": 112918528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048776328986960884, + "loss": 3.3579, + "theoretical_loss": 4.692810147116814, + "tokens_seen": 112984064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 72808, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.543510913848877, + "objective/train/theoretical_loss": 4.692439347328926, + "objective/train/tokens_used": 133509600, + "theoretical_loss": 4.692439347328926, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487753259779338, + "loss": 3.5704, + "theoretical_loss": 4.692439347328926, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877432296890672, + "loss": 3.2623, + "theoretical_loss": 4.692068822583197, + "tokens_seen": 113115136 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048773319959879644, + "loss": 3.7086, + "theoretical_loss": 4.6916985725164375, + "tokens_seen": 113180672 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048772316950852556, + "loss": 3.2168, + "theoretical_loss": 4.691328596766148, + "tokens_seen": 113246208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877131394182548, + "loss": 3.5721, + "theoretical_loss": 4.690958894970518, + "tokens_seen": 113311744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877031093279839, + "loss": 3.5405, + "theoretical_loss": 4.690589466768422, + "tokens_seen": 113377280 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048769307923771316, + "loss": 3.4867, + "theoretical_loss": 4.69022031179942, + "tokens_seen": 113442816 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048768304914744234, + "loss": 3.3823, + "theoretical_loss": 4.689851429703753, + "tokens_seen": 113508352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876730190571715, + "loss": 3.242, + "theoretical_loss": 4.689482820122342, + "tokens_seen": 113573888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876629889669007, + "loss": 3.6015, + "theoretical_loss": 4.689114482696793, + "tokens_seen": 113639424 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048765295887662994, + "loss": 3.4168, + "theoretical_loss": 4.6887464170693836, + "tokens_seen": 113704960 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048764292878635907, + "loss": 3.4562, + "theoretical_loss": 4.6883786228830715, + "tokens_seen": 113770496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876328986960883, + "loss": 3.1948, + "theoretical_loss": 4.688011099781486, + "tokens_seen": 113836032 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048762286860581743, + "loss": 3.2999, + "theoretical_loss": 4.687643847408929, + "tokens_seen": 113901568 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048761283851554666, + "loss": 3.342, + "theoretical_loss": 4.687276865410377, + "tokens_seen": 113967104 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048760280842527584, + "loss": 3.4757, + "theoretical_loss": 4.686910153431472, + "tokens_seen": 114032640 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487592778335005, + "loss": 3.5307, + "theoretical_loss": 4.686543711118526, + "tokens_seen": 114098176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875827482447342, + "loss": 3.2597, + "theoretical_loss": 4.686177538118516, + "tokens_seen": 114163712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875727181544634, + "loss": 3.4569, + "theoretical_loss": 4.685811634079084, + "tokens_seen": 114229248 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048756268806419257, + "loss": 3.3531, + "theoretical_loss": 4.685445998648532, + "tokens_seen": 114294784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875526579739218, + "loss": 3.4034, + "theoretical_loss": 4.685080631475831, + "tokens_seen": 114360320 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048754262788365093, + "loss": 3.0558, + "theoretical_loss": 4.6847155322106016, + "tokens_seen": 114425856 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048753259779338017, + "loss": 3.4359, + "theoretical_loss": 4.6843507005031295, + "tokens_seen": 114491392 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048752256770310935, + "loss": 3.4293, + "theoretical_loss": 4.683986136004355, + "tokens_seen": 114556928 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048751253761283853, + "loss": 3.1354, + "theoretical_loss": 4.6836218383658705, + "tokens_seen": 114622464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 73349, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8527109622955322, + "objective/train/theoretical_loss": 4.683257807239925, + "objective/train/tokens_used": 135148000, + "theoretical_loss": 4.683257807239925, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875025075225677, + "loss": 3.1073, + "theoretical_loss": 4.683257807239925, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874924774322969, + "loss": 3.096, + "theoretical_loss": 4.682894042279417, + "tokens_seen": 114753536 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048748244734202607, + "loss": 3.2853, + "theoretical_loss": 4.682530543137897, + "tokens_seen": 114819072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874724172517553, + "loss": 3.6298, + "theoretical_loss": 4.68216730946956, + "tokens_seen": 114884608 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048746238716148443, + "loss": 3.305, + "theoretical_loss": 4.6818043409292525, + "tokens_seen": 114950144 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048745235707121367, + "loss": 3.4946, + "theoretical_loss": 4.681441637172463, + "tokens_seen": 115015680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874423269809428, + "loss": 3.3972, + "theoretical_loss": 4.681079197855325, + "tokens_seen": 115081216 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048743229689067203, + "loss": 3.1421, + "theoretical_loss": 4.680717022634612, + "tokens_seen": 115146752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874222668004012, + "loss": 3.6661, + "theoretical_loss": 4.680355111167742, + "tokens_seen": 115212288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874122367101304, + "loss": 3.2117, + "theoretical_loss": 4.679993463112769, + "tokens_seen": 115277824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874022066198596, + "loss": 3.4083, + "theoretical_loss": 4.679632078128385, + "tokens_seen": 115343360 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048739217652958876, + "loss": 3.2653, + "theoretical_loss": 4.679270955873918, + "tokens_seen": 115408896 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048738214643931794, + "loss": 3.4833, + "theoretical_loss": 4.678910096009329, + "tokens_seen": 115474432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048737211634904717, + "loss": 3.3117, + "theoretical_loss": 4.678549498195213, + "tokens_seen": 115539968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873620862587763, + "loss": 3.3255, + "theoretical_loss": 4.678189162092798, + "tokens_seen": 115605504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048735205616850553, + "loss": 3.2552, + "theoretical_loss": 4.677829087363937, + "tokens_seen": 115671040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873420260782347, + "loss": 3.2912, + "theoretical_loss": 4.677469273671118, + "tokens_seen": 115736576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873319959879639, + "loss": 3.1695, + "theoretical_loss": 4.677109720677448, + "tokens_seen": 115802112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873219658976931, + "loss": 3.2226, + "theoretical_loss": 4.676750428046665, + "tokens_seen": 115867648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048731193580742226, + "loss": 3.2803, + "theoretical_loss": 4.676391395443128, + "tokens_seen": 115933184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048730190571715144, + "loss": 3.5568, + "theoretical_loss": 4.6760326225318165, + "tokens_seen": 115998720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872918756268807, + "loss": 3.339, + "theoretical_loss": 4.675674108978335, + "tokens_seen": 116064256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872818455366098, + "loss": 3.1442, + "theoretical_loss": 4.675315854448903, + "tokens_seen": 116129792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048727181544633904, + "loss": 3.1936, + "theoretical_loss": 4.674957858610361, + "tokens_seen": 116195328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048726178535606816, + "loss": 3.2333, + "theoretical_loss": 4.674600121130163, + "tokens_seen": 116260864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 74658, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9736719131469727, + "objective/train/theoretical_loss": 4.674242641676377, + "objective/train/tokens_used": 136786400, + "theoretical_loss": 4.674242641676377, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872517552657974, + "loss": 3.3693, + "theoretical_loss": 4.674242641676377, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872417251755266, + "loss": 3.3527, + "theoretical_loss": 4.673885419917689, + "tokens_seen": 116391936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048723169508525576, + "loss": 3.5978, + "theoretical_loss": 4.673528455523392, + "tokens_seen": 116457472 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487221664994985, + "loss": 3.416, + "theoretical_loss": 4.673171748163389, + "tokens_seen": 116523008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872116349047141, + "loss": 3.3723, + "theoretical_loss": 4.672815297508194, + "tokens_seen": 116588544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048720160481444336, + "loss": 3.1412, + "theoretical_loss": 4.672459103228929, + "tokens_seen": 116654080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048719157472417254, + "loss": 3.6399, + "theoretical_loss": 4.672103164997319, + "tokens_seen": 116719616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871815446339017, + "loss": 3.3651, + "theoretical_loss": 4.671747482485696, + "tokens_seen": 116785152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871715145436309, + "loss": 3.3461, + "theoretical_loss": 4.671392055366992, + "tokens_seen": 116850688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048716148445336014, + "loss": 3.2057, + "theoretical_loss": 4.671036883314743, + "tokens_seen": 116916224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048715145436308927, + "loss": 3.1564, + "theoretical_loss": 4.670681966003084, + "tokens_seen": 116981760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871414242728185, + "loss": 3.3224, + "theoretical_loss": 4.670327303106748, + "tokens_seen": 117047296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048713139418254763, + "loss": 3.5285, + "theoretical_loss": 4.6699728943010665, + "tokens_seen": 117112832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048712136409227686, + "loss": 3.2677, + "theoretical_loss": 4.669618739261967, + "tokens_seen": 117178368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048711133400200604, + "loss": 3.1055, + "theoretical_loss": 4.669264837665969, + "tokens_seen": 117243904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871013039117352, + "loss": 3.3293, + "theoretical_loss": 4.668911189190185, + "tokens_seen": 117309440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870912738214644, + "loss": 3.3489, + "theoretical_loss": 4.668557793512322, + "tokens_seen": 117374976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870812437311936, + "loss": 3.1125, + "theoretical_loss": 4.668204650310676, + "tokens_seen": 117440512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048707121364092277, + "loss": 3.2515, + "theoretical_loss": 4.667851759264128, + "tokens_seen": 117506048 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487061183550652, + "loss": 3.3586, + "theoretical_loss": 4.667499120052151, + "tokens_seen": 117571584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048705115346038113, + "loss": 3.2803, + "theoretical_loss": 4.6671467323548, + "tokens_seen": 117637120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048704112337011037, + "loss": 3.3058, + "theoretical_loss": 4.666794595852718, + "tokens_seen": 117702656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048703109327983955, + "loss": 3.2752, + "theoretical_loss": 4.666442710227129, + "tokens_seen": 117768192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048702106318956873, + "loss": 3.3295, + "theoretical_loss": 4.666091075159839, + "tokens_seen": 117833728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870110330992979, + "loss": 3.5682, + "theoretical_loss": 4.665739690333233, + "tokens_seen": 117899264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 75891, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9445178508758545, + "objective/train/theoretical_loss": 4.665388555430277, + "objective/train/tokens_used": 138424800, + "theoretical_loss": 4.665388555430277, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870010030090271, + "loss": 3.4408, + "theoretical_loss": 4.665388555430277, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048699097291875627, + "loss": 3.42, + "theoretical_loss": 4.665037670134513, + "tokens_seen": 118030336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869809428284855, + "loss": 3.4118, + "theoretical_loss": 4.66468703413006, + "tokens_seen": 118095872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048697091273821463, + "loss": 2.8531, + "theoretical_loss": 4.664336647101612, + "tokens_seen": 118161408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048696088264794387, + "loss": 3.5486, + "theoretical_loss": 4.663986508734435, + "tokens_seen": 118226944 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486950852557673, + "loss": 3.3137, + "theoretical_loss": 4.663636618714368, + "tokens_seen": 118292480 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048694082246740223, + "loss": 3.2493, + "theoretical_loss": 4.663286976727821, + "tokens_seen": 118358016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869307923771314, + "loss": 3.4941, + "theoretical_loss": 4.6629375824617725, + "tokens_seen": 118423552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869207622868606, + "loss": 3.1022, + "theoretical_loss": 4.6625884356037695, + "tokens_seen": 118489088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869107321965898, + "loss": 3.2497, + "theoretical_loss": 4.662239535841925, + "tokens_seen": 118554624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048690070210631896, + "loss": 3.465, + "theoretical_loss": 4.661890882864919, + "tokens_seen": 118620160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048689067201604814, + "loss": 3.3506, + "theoretical_loss": 4.661542476361992, + "tokens_seen": 118685696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688064192577737, + "loss": 3.2192, + "theoretical_loss": 4.661194316022952, + "tokens_seen": 118751232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868706118355065, + "loss": 3.274, + "theoretical_loss": 4.660846401538165, + "tokens_seen": 118816768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686058174523573, + "loss": 3.1325, + "theoretical_loss": 4.6604987325985565, + "tokens_seen": 118882304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868505516549649, + "loss": 3.241, + "theoretical_loss": 4.660151308895614, + "tokens_seen": 118947840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868405215646941, + "loss": 3.457, + "theoretical_loss": 4.659804130121378, + "tokens_seen": 119013376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868304914744233, + "loss": 3.2732, + "theoretical_loss": 4.659457195968448, + "tokens_seen": 119078912 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682046138415246, + "loss": 3.1738, + "theoretical_loss": 4.659110506129977, + "tokens_seen": 119144448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048681043129388164, + "loss": 3.2882, + "theoretical_loss": 4.6587640602996725, + "tokens_seen": 119209984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868004012036109, + "loss": 3.4992, + "theoretical_loss": 4.658417858171794, + "tokens_seen": 119275520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048679037111334, + "loss": 3.2217, + "theoretical_loss": 4.6580718994411505, + "tokens_seen": 119341056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678034102306924, + "loss": 3.3294, + "theoretical_loss": 4.657726183803101, + "tokens_seen": 119406592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048677031093279836, + "loss": 3.0235, + "theoretical_loss": 4.657380710953554, + "tokens_seen": 119472128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867602808425276, + "loss": 3.2772, + "theoretical_loss": 4.657035480588962, + "tokens_seen": 119537664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 76546, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.836437225341797, + "objective/train/theoretical_loss": 4.656690492406327, + "objective/train/tokens_used": 140063200, + "theoretical_loss": 4.656690492406327, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867502507522568, + "loss": 3.395, + "theoretical_loss": 4.656690492406327, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048674022066198596, + "loss": 3.2332, + "theoretical_loss": 4.6563457461031925, + "tokens_seen": 119668736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048673019057171514, + "loss": 3.3067, + "theoretical_loss": 4.656001241377647, + "tokens_seen": 119734272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867201604814443, + "loss": 3.4375, + "theoretical_loss": 4.655656977928318, + "tokens_seen": 119799808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867101303911735, + "loss": 3.3728, + "theoretical_loss": 4.655312955454378, + "tokens_seen": 119865344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048670010030090274, + "loss": 3.4504, + "theoretical_loss": 4.654969173655536, + "tokens_seen": 119930880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048669007021063187, + "loss": 3.168, + "theoretical_loss": 4.654625632232037, + "tokens_seen": 119996416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866800401203611, + "loss": 3.1419, + "theoretical_loss": 4.654282330884667, + "tokens_seen": 120061952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866700100300903, + "loss": 3.483, + "theoretical_loss": 4.653939269314744, + "tokens_seen": 120127488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048665997993981947, + "loss": 3.1995, + "theoretical_loss": 4.653596447224125, + "tokens_seen": 120193024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048664994984954865, + "loss": 3.0174, + "theoretical_loss": 4.653253864315193, + "tokens_seen": 120258560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048663991975927783, + "loss": 3.2561, + "theoretical_loss": 4.6529115202908695, + "tokens_seen": 120324096 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486629889669007, + "loss": 3.3328, + "theoretical_loss": 4.652569414854604, + "tokens_seen": 120389632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048661985957873624, + "loss": 3.2924, + "theoretical_loss": 4.652227547710373, + "tokens_seen": 120455168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048660982948846537, + "loss": 3.2271, + "theoretical_loss": 4.651885918562684, + "tokens_seen": 120520704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865997993981946, + "loss": 3.2564, + "theoretical_loss": 4.65154452711657, + "tokens_seen": 120586240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048658976930792373, + "loss": 3.2625, + "theoretical_loss": 4.651203373077593, + "tokens_seen": 120651776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048657973921765297, + "loss": 3.1227, + "theoretical_loss": 4.650862456151833, + "tokens_seen": 120717312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048656970912738215, + "loss": 3.2546, + "theoretical_loss": 4.650521776045898, + "tokens_seen": 120782848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048655967903711133, + "loss": 3.2324, + "theoretical_loss": 4.650181332466918, + "tokens_seen": 120848384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865496489468405, + "loss": 3.3792, + "theoretical_loss": 4.64984112512254, + "tokens_seen": 120913920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048653961885656975, + "loss": 3.4118, + "theoretical_loss": 4.6495011537209345, + "tokens_seen": 120979456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865295887662989, + "loss": 3.3581, + "theoretical_loss": 4.649161417970789, + "tokens_seen": 121044992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865195586760281, + "loss": 3.4152, + "theoretical_loss": 4.648821917581308, + "tokens_seen": 121110528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048650952858575724, + "loss": 3.4244, + "theoretical_loss": 4.648482652262211, + "tokens_seen": 121176064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 77942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9632604122161865, + "objective/train/theoretical_loss": 4.648143621723734, + "objective/train/tokens_used": 141701600, + "theoretical_loss": 4.648143621723734, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048649949849548647, + "loss": 3.2822, + "theoretical_loss": 4.648143621723734, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048648946840521565, + "loss": 3.0832, + "theoretical_loss": 4.6478048256766265, + "tokens_seen": 121307136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048647943831494483, + "loss": 3.3079, + "theoretical_loss": 4.647466263832148, + "tokens_seen": 121372672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048646940822467407, + "loss": 3.2723, + "theoretical_loss": 4.647127935902072, + "tokens_seen": 121438208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864593781344032, + "loss": 3.5748, + "theoretical_loss": 4.646789841598681, + "tokens_seen": 121503744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048644934804413243, + "loss": 3.1642, + "theoretical_loss": 4.646451980634765, + "tokens_seen": 121569280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864393179538616, + "loss": 3.1145, + "theoretical_loss": 4.646114352723625, + "tokens_seen": 121634816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864292878635908, + "loss": 3.5071, + "theoretical_loss": 4.645776957579065, + "tokens_seen": 121700352 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048641925777332, + "loss": 3.4015, + "theoretical_loss": 4.645439794915395, + "tokens_seen": 121765888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048640922768304916, + "loss": 3.2914, + "theoretical_loss": 4.6451028644474315, + "tokens_seen": 121831424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048639919759277834, + "loss": 3.2374, + "theoretical_loss": 4.644766165890491, + "tokens_seen": 121896960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048638916750250757, + "loss": 3.2349, + "theoretical_loss": 4.644429698960394, + "tokens_seen": 121962496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863791374122367, + "loss": 3.4859, + "theoretical_loss": 4.644093463373462, + "tokens_seen": 122028032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048636910732196593, + "loss": 3.2108, + "theoretical_loss": 4.643757458846514, + "tokens_seen": 122093568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863590772316951, + "loss": 3.2694, + "theoretical_loss": 4.643421685096866, + "tokens_seen": 122159104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863490471414243, + "loss": 3.525, + "theoretical_loss": 4.643086141842337, + "tokens_seen": 122224640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863390170511535, + "loss": 3.4927, + "theoretical_loss": 4.642750828801238, + "tokens_seen": 122290176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048632898696088266, + "loss": 3.4461, + "theoretical_loss": 4.642415745692375, + "tokens_seen": 122355712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048631895687061184, + "loss": 3.3788, + "theoretical_loss": 4.6420808922350485, + "tokens_seen": 122421248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863089267803411, + "loss": 3.3053, + "theoretical_loss": 4.641746268149052, + "tokens_seen": 122486784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862988966900702, + "loss": 3.2416, + "theoretical_loss": 4.64141187315467, + "tokens_seen": 122552320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048628886659979944, + "loss": 3.0169, + "theoretical_loss": 4.641077706972679, + "tokens_seen": 122617856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048627883650952857, + "loss": 3.3138, + "theoretical_loss": 4.6407437693243425, + "tokens_seen": 122683392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862688064192578, + "loss": 3.1577, + "theoretical_loss": 4.640410059931414, + "tokens_seen": 122748928 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486258776328987, + "loss": 3.3863, + "theoretical_loss": 4.640076578516132, + "tokens_seen": 122814464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 78541, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7341437339782715, + "objective/train/theoretical_loss": 4.639743324801225, + "objective/train/tokens_used": 143340000, + "theoretical_loss": 4.639743324801225, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048624874623871616, + "loss": 3.1381, + "theoretical_loss": 4.639743324801225, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048623871614844534, + "loss": 3.3316, + "theoretical_loss": 4.639410298509901, + "tokens_seen": 122945536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862286860581745, + "loss": 3.2356, + "theoretical_loss": 4.639077499365856, + "tokens_seen": 123011072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862186559679037, + "loss": 3.355, + "theoretical_loss": 4.638744927093268, + "tokens_seen": 123076608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048620862587763294, + "loss": 3.3971, + "theoretical_loss": 4.6384125814167945, + "tokens_seen": 123142144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048619859578736207, + "loss": 3.0734, + "theoretical_loss": 4.638080462061575, + "tokens_seen": 123207680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861885656970913, + "loss": 3.1079, + "theoretical_loss": 4.63774856875323, + "tokens_seen": 123273216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861785356068205, + "loss": 3.3234, + "theoretical_loss": 4.637416901217852, + "tokens_seen": 123338752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048616850551654967, + "loss": 3.3152, + "theoretical_loss": 4.63708545918202, + "tokens_seen": 123404288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048615847542627885, + "loss": 3.345, + "theoretical_loss": 4.636754242372782, + "tokens_seen": 123469824 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048614844533600803, + "loss": 3.3265, + "theoretical_loss": 4.636423250517664, + "tokens_seen": 123535360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861384152457372, + "loss": 3.3053, + "theoretical_loss": 4.636092483344665, + "tokens_seen": 123600896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048612838515546644, + "loss": 3.3878, + "theoretical_loss": 4.635761940582256, + "tokens_seen": 123666432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048611835506519557, + "loss": 3.3957, + "theoretical_loss": 4.635431621959384, + "tokens_seen": 123731968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861083249749248, + "loss": 3.4768, + "theoretical_loss": 4.63510152720546, + "tokens_seen": 123797504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048609829488465393, + "loss": 3.2736, + "theoretical_loss": 4.634771656050372, + "tokens_seen": 123863040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048608826479438317, + "loss": 3.1239, + "theoretical_loss": 4.63444200822447, + "tokens_seen": 123928576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048607823470411235, + "loss": 3.2963, + "theoretical_loss": 4.6341125834585775, + "tokens_seen": 123994112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048606820461384153, + "loss": 3.4747, + "theoretical_loss": 4.633783381483978, + "tokens_seen": 124059648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860581745235707, + "loss": 3.374, + "theoretical_loss": 4.633454402032427, + "tokens_seen": 124125184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048604814443329995, + "loss": 3.4227, + "theoretical_loss": 4.63312564483614, + "tokens_seen": 124190720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860381143430291, + "loss": 3.6141, + "theoretical_loss": 4.632797109627798, + "tokens_seen": 124256256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860280842527583, + "loss": 3.2804, + "theoretical_loss": 4.632468796140543, + "tokens_seen": 124321792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048601805416248744, + "loss": 3.4935, + "theoretical_loss": 4.632140704107979, + "tokens_seen": 124387328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048600802407221667, + "loss": 3.3112, + "theoretical_loss": 4.631812833264169, + "tokens_seen": 124452864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 79709, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.385240316390991, + "objective/train/theoretical_loss": 4.631485183343639, + "objective/train/tokens_used": 144978400, + "theoretical_loss": 4.631485183343639, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048599799398194585, + "loss": 3.2349, + "theoretical_loss": 4.631485183343639, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048598796389167503, + "loss": 3.3546, + "theoretical_loss": 4.631157754081366, + "tokens_seen": 124583936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859779338014042, + "loss": 3.088, + "theoretical_loss": 4.630830545212792, + "tokens_seen": 124649472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859679037111334, + "loss": 3.4382, + "theoretical_loss": 4.63050355647381, + "tokens_seen": 124715008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859578736208626, + "loss": 3.3753, + "theoretical_loss": 4.630176787600771, + "tokens_seen": 124780544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859478435305918, + "loss": 3.1749, + "theoretical_loss": 4.6298502383304765, + "tokens_seen": 124846080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048593781344032094, + "loss": 3.1883, + "theoretical_loss": 4.629523908400184, + "tokens_seen": 124911616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859277833500502, + "loss": 3.4012, + "theoretical_loss": 4.6291977975476035, + "tokens_seen": 124977152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859177532597793, + "loss": 3.1556, + "theoretical_loss": 4.628871905510893, + "tokens_seen": 125042688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048590772316950854, + "loss": 3.5815, + "theoretical_loss": 4.628546232028661, + "tokens_seen": 125108224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858976930792377, + "loss": 3.2682, + "theoretical_loss": 4.62822077683997, + "tokens_seen": 125173760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858876629889669, + "loss": 3.685, + "theoretical_loss": 4.627895539684323, + "tokens_seen": 125239296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858776328986961, + "loss": 3.2897, + "theoretical_loss": 4.627570520301673, + "tokens_seen": 125304832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858676028084253, + "loss": 3.1734, + "theoretical_loss": 4.627245718432423, + "tokens_seen": 125370368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048585757271815444, + "loss": 3.0723, + "theoretical_loss": 4.626921133817413, + "tokens_seen": 125435904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858475426278837, + "loss": 3.1019, + "theoretical_loss": 4.626596766197936, + "tokens_seen": 125501440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858375125376128, + "loss": 3.21, + "theoretical_loss": 4.62627261531572, + "tokens_seen": 125566976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048582748244734204, + "loss": 3.1586, + "theoretical_loss": 4.625948680912939, + "tokens_seen": 125632512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858174523570712, + "loss": 3.1464, + "theoretical_loss": 4.625624962732207, + "tokens_seen": 125698048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858074222668004, + "loss": 3.5956, + "theoretical_loss": 4.62530146051658, + "tokens_seen": 125763584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857973921765296, + "loss": 3.2982, + "theoretical_loss": 4.624978174009549, + "tokens_seen": 125829120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048578736208625877, + "loss": 2.8431, + "theoretical_loss": 4.624655102955048, + "tokens_seen": 125894656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577733199598795, + "loss": 3.2039, + "theoretical_loss": 4.624332247097442, + "tokens_seen": 125960192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857673019057172, + "loss": 3.255, + "theoretical_loss": 4.624009606181539, + "tokens_seen": 126025728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857572718154463, + "loss": 3.3476, + "theoretical_loss": 4.623687179952577, + "tokens_seen": 126091264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 80984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0424399375915527, + "objective/train/theoretical_loss": 4.623364968156229, + "objective/train/tokens_used": 146616800, + "theoretical_loss": 4.623364968156229, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048574724172517554, + "loss": 3.1198, + "theoretical_loss": 4.623364968156229, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573721163490467, + "loss": 3.2627, + "theoretical_loss": 4.623042970538604, + "tokens_seen": 126222336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857271815446339, + "loss": 3.2619, + "theoretical_loss": 4.622721186846241, + "tokens_seen": 126287872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048571715145436314, + "loss": 3.077, + "theoretical_loss": 4.62239961682611, + "tokens_seen": 126353408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048570712136409227, + "loss": 3.4283, + "theoretical_loss": 4.622078260225611, + "tokens_seen": 126418944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856970912738215, + "loss": 3.1504, + "theoretical_loss": 4.621757116792576, + "tokens_seen": 126484480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856870611835507, + "loss": 3.2759, + "theoretical_loss": 4.621436186275262, + "tokens_seen": 126550016 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048567703109327987, + "loss": 3.2942, + "theoretical_loss": 4.6211154684223565, + "tokens_seen": 126615552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048566700100300905, + "loss": 3.462, + "theoretical_loss": 4.6207949629829725, + "tokens_seen": 126681088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048565697091273823, + "loss": 2.9686, + "theoretical_loss": 4.620474669706647, + "tokens_seen": 126746624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856469408224674, + "loss": 3.3596, + "theoretical_loss": 4.620154588343343, + "tokens_seen": 126812160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563691073219664, + "loss": 3.3793, + "theoretical_loss": 4.619834718643448, + "tokens_seen": 126877696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048562688064192577, + "loss": 3.327, + "theoretical_loss": 4.619515060357771, + "tokens_seen": 126943232 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485616850551655, + "loss": 3.5938, + "theoretical_loss": 4.619195613237542, + "tokens_seen": 127008768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048560682046138413, + "loss": 3.1359, + "theoretical_loss": 4.6188763770344154, + "tokens_seen": 127074304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559679037111337, + "loss": 3.6939, + "theoretical_loss": 4.618557351500461, + "tokens_seen": 127139840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048558676028084255, + "loss": 3.3604, + "theoretical_loss": 4.618238536388171, + "tokens_seen": 127205376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048557673019057173, + "loss": 3.4903, + "theoretical_loss": 4.617919931450454, + "tokens_seen": 127270912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855667001003009, + "loss": 3.3289, + "theoretical_loss": 4.617601536440637, + "tokens_seen": 127336448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048555667001003015, + "loss": 3.1247, + "theoretical_loss": 4.617283351112462, + "tokens_seen": 127401984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855466399197593, + "loss": 3.5764, + "theoretical_loss": 4.616965375220088, + "tokens_seen": 127467520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855366098294885, + "loss": 3.419, + "theoretical_loss": 4.616647608518086, + "tokens_seen": 127533056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048552657973921764, + "loss": 3.2447, + "theoretical_loss": 4.616330050761444, + "tokens_seen": 127598592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048551654964894687, + "loss": 3.0136, + "theoretical_loss": 4.616012701705559, + "tokens_seen": 127664128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048550651955867605, + "loss": 3.1339, + "theoretical_loss": 4.615695561106241, + "tokens_seen": 127729664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 81703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6864616870880127, + "objective/train/theoretical_loss": 4.6153786287197125, + "objective/train/tokens_used": 148255200, + "theoretical_loss": 4.6153786287197125, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048549648946840523, + "loss": 3.2948, + "theoretical_loss": 4.6153786287197125, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854864593781344, + "loss": 3.0792, + "theoretical_loss": 4.615061904302603, + "tokens_seen": 127860736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854764292878636, + "loss": 3.1573, + "theoretical_loss": 4.614745387611955, + "tokens_seen": 127926272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854663991975928, + "loss": 3.4941, + "theoretical_loss": 4.614429078405214, + "tokens_seen": 127991808 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485456369107322, + "loss": 3.2639, + "theoretical_loss": 4.6141129764402375, + "tokens_seen": 128057344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048544633901705114, + "loss": 3.2058, + "theoretical_loss": 4.613797081475287, + "tokens_seen": 128122880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854363089267804, + "loss": 3.4029, + "theoretical_loss": 4.613481393269028, + "tokens_seen": 128188416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854262788365095, + "loss": 3.5993, + "theoretical_loss": 4.613165911580533, + "tokens_seen": 128253952 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048541624874623874, + "loss": 3.1374, + "theoretical_loss": 4.612850636169279, + "tokens_seen": 128319488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854062186559679, + "loss": 3.3721, + "theoretical_loss": 4.612535566795142, + "tokens_seen": 128385024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853961885656971, + "loss": 3.1656, + "theoretical_loss": 4.612220703218403, + "tokens_seen": 128450560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853861584754263, + "loss": 3.4822, + "theoretical_loss": 4.611906045199744, + "tokens_seen": 128516096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853761283851555, + "loss": 3.1366, + "theoretical_loss": 4.611591592500245, + "tokens_seen": 128581632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048536609829488464, + "loss": 3.3858, + "theoretical_loss": 4.611277344881389, + "tokens_seen": 128647168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853560682046139, + "loss": 3.1729, + "theoretical_loss": 4.610963302105051, + "tokens_seen": 128712704 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485346038114343, + "loss": 3.2684, + "theoretical_loss": 4.610649463933511, + "tokens_seen": 128778240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533600802407224, + "loss": 3.1187, + "theoretical_loss": 4.61033583012944, + "tokens_seen": 128843776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853259779338014, + "loss": 3.2702, + "theoretical_loss": 4.610022400455909, + "tokens_seen": 128909312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853159478435306, + "loss": 3.1401, + "theoretical_loss": 4.6097091746763805, + "tokens_seen": 128974848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853059177532598, + "loss": 3.0625, + "theoretical_loss": 4.609396152554714, + "tokens_seen": 129040384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048529588766298897, + "loss": 3.4469, + "theoretical_loss": 4.60908333385516, + "tokens_seen": 129105920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048528585757271815, + "loss": 3.2416, + "theoretical_loss": 4.608770718342361, + "tokens_seen": 129171456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852758274824474, + "loss": 3.2241, + "theoretical_loss": 4.6084583057813555, + "tokens_seen": 129236992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852657973921765, + "loss": 3.2384, + "theoretical_loss": 4.608146095937567, + "tokens_seen": 129302528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525576730190574, + "loss": 3.4009, + "theoretical_loss": 4.607834088576811, + "tokens_seen": 129368064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 82831, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1748886108398438, + "objective/train/theoretical_loss": 4.607522283465294, + "objective/train/tokens_used": 149893600, + "theoretical_loss": 4.607522283465294, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048524573721163487, + "loss": 3.2513, + "theoretical_loss": 4.607522283465294, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852357071213641, + "loss": 3.3024, + "theoretical_loss": 4.607210680369608, + "tokens_seen": 129499136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852256770310933, + "loss": 3.5175, + "theoretical_loss": 4.606899279056734, + "tokens_seen": 129564672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521564694082247, + "loss": 3.1854, + "theoretical_loss": 4.606588079294038, + "tokens_seen": 129630208 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048520561685055165, + "loss": 3.1448, + "theoretical_loss": 4.606277080849273, + "tokens_seen": 129695744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851955867602809, + "loss": 3.2784, + "theoretical_loss": 4.605966283490575, + "tokens_seen": 129761280 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048518555667001, + "loss": 3.055, + "theoretical_loss": 4.6056556869864655, + "tokens_seen": 129826816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048517552657973925, + "loss": 3.4625, + "theoretical_loss": 4.60534529110585, + "tokens_seen": 129892352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851654964894684, + "loss": 3.2055, + "theoretical_loss": 4.605035095618014, + "tokens_seen": 129957888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851554663991976, + "loss": 3.3952, + "theoretical_loss": 4.604725100292626, + "tokens_seen": 130023424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851454363089268, + "loss": 3.193, + "theoretical_loss": 4.604415304899735, + "tokens_seen": 130088960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513540621865597, + "loss": 3.3349, + "theoretical_loss": 4.604105709209769, + "tokens_seen": 130154496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048512537612838515, + "loss": 3.4078, + "theoretical_loss": 4.603796312993536, + "tokens_seen": 130220032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048511534603811433, + "loss": 3.3374, + "theoretical_loss": 4.603487116022222, + "tokens_seen": 130285568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851053159478435, + "loss": 3.2205, + "theoretical_loss": 4.60317811806739, + "tokens_seen": 130351104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048509528585757275, + "loss": 3.3053, + "theoretical_loss": 4.60286931890098, + "tokens_seen": 130416640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850852557673019, + "loss": 3.3346, + "theoretical_loss": 4.602560718295308, + "tokens_seen": 130482176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850752256770311, + "loss": 3.3864, + "theoretical_loss": 4.602252316023064, + "tokens_seen": 130547712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048506519558676024, + "loss": 3.4327, + "theoretical_loss": 4.601944111857312, + "tokens_seen": 130613248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850551654964895, + "loss": 3.2583, + "theoretical_loss": 4.601636105571492, + "tokens_seen": 130678784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048504513540621866, + "loss": 3.4605, + "theoretical_loss": 4.601328296939414, + "tokens_seen": 130744320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503510531594784, + "loss": 3.2544, + "theoretical_loss": 4.601020685735259, + "tokens_seen": 130809856 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485025075225677, + "loss": 3.2836, + "theoretical_loss": 4.600713271733582, + "tokens_seen": 130875392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048501504513540625, + "loss": 3.4586, + "theoretical_loss": 4.600406054709307, + "tokens_seen": 130940928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850050150451354, + "loss": 3.3115, + "theoretical_loss": 4.600099034437726, + "tokens_seen": 131006464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 83204, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2750132083892822, + "objective/train/theoretical_loss": 4.5997922106945, + "objective/train/tokens_used": 151532000, + "theoretical_loss": 4.5997922106945, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849949849548646, + "loss": 3.3318, + "theoretical_loss": 4.5997922106945, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849849548645938, + "loss": 3.5144, + "theoretical_loss": 4.5994855832556585, + "tokens_seen": 131137536 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484974924774323, + "loss": 3.4244, + "theoretical_loss": 4.5991791518976, + "tokens_seen": 131203072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849648946840522, + "loss": 3.2389, + "theoretical_loss": 4.5988729163970845, + "tokens_seen": 131268608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048495486459378134, + "loss": 3.3639, + "theoretical_loss": 4.59856687653124, + "tokens_seen": 131334144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849448345035106, + "loss": 3.2156, + "theoretical_loss": 4.598261032077561, + "tokens_seen": 131399680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849348044132397, + "loss": 3.5008, + "theoretical_loss": 4.597955382813903, + "tokens_seen": 131465216 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048492477432296894, + "loss": 3.4594, + "theoretical_loss": 4.597649928518486, + "tokens_seen": 131530752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849147442326981, + "loss": 3.3812, + "theoretical_loss": 4.597344668969892, + "tokens_seen": 131596288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849047141424273, + "loss": 3.3369, + "theoretical_loss": 4.597039603947064, + "tokens_seen": 131661824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848946840521565, + "loss": 3.2768, + "theoretical_loss": 4.596734733229306, + "tokens_seen": 131727360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848846539618857, + "loss": 2.9459, + "theoretical_loss": 4.596430056596285, + "tokens_seen": 131792896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048487462387161484, + "loss": 3.0683, + "theoretical_loss": 4.596125573828021, + "tokens_seen": 131858432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848645937813441, + "loss": 3.3636, + "theoretical_loss": 4.595821284704899, + "tokens_seen": 131923968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848545636910732, + "loss": 3.2567, + "theoretical_loss": 4.5955171890076585, + "tokens_seen": 131989504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048484453360080244, + "loss": 3.5107, + "theoretical_loss": 4.595213286517394, + "tokens_seen": 132055040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848345035105316, + "loss": 3.3222, + "theoretical_loss": 4.594909577015562, + "tokens_seen": 132120576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848244734202608, + "loss": 3.4088, + "theoretical_loss": 4.594606060283969, + "tokens_seen": 132186112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048481444332999, + "loss": 3.3058, + "theoretical_loss": 4.594302736104778, + "tokens_seen": 132251648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048480441323971917, + "loss": 3.3761, + "theoretical_loss": 4.593999604260507, + "tokens_seen": 132317184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048479438314944835, + "loss": 3.115, + "theoretical_loss": 4.593696664534026, + "tokens_seen": 132382720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847843530591776, + "loss": 3.0734, + "theoretical_loss": 4.593393916708557, + "tokens_seen": 132448256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847743229689067, + "loss": 3.4007, + "theoretical_loss": 4.593091360567676, + "tokens_seen": 132513792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048476429287863594, + "loss": 3.4221, + "theoretical_loss": 4.59278899589531, + "tokens_seen": 132579328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048475426278836507, + "loss": 3.3316, + "theoretical_loss": 4.592486822475731, + "tokens_seen": 132644864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 84183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4309723377227783, + "objective/train/theoretical_loss": 4.592184840093566, + "objective/train/tokens_used": 153170400, + "theoretical_loss": 4.592184840093566, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847442326980943, + "loss": 3.1609, + "theoretical_loss": 4.592184840093566, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847342026078235, + "loss": 3.3487, + "theoretical_loss": 4.59188304853379, + "tokens_seen": 132775936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048472417251755267, + "loss": 3.2624, + "theoretical_loss": 4.591581447581723, + "tokens_seen": 132841472 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048471414242728185, + "loss": 3.0568, + "theoretical_loss": 4.591280037023036, + "tokens_seen": 132907008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847041123370111, + "loss": 3.3944, + "theoretical_loss": 4.590978816643744, + "tokens_seen": 132972544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846940822467402, + "loss": 3.375, + "theoretical_loss": 4.590677786230208, + "tokens_seen": 133038080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048468405215646945, + "loss": 3.1915, + "theoretical_loss": 4.590376945569136, + "tokens_seen": 133103616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846740220661986, + "loss": 3.2734, + "theoretical_loss": 4.590076294447578, + "tokens_seen": 133169152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846639919759278, + "loss": 3.1604, + "theoretical_loss": 4.589775832652929, + "tokens_seen": 133234688 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484653961885657, + "loss": 3.1837, + "theoretical_loss": 4.589475559972927, + "tokens_seen": 133300224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048464393179538617, + "loss": 3.3031, + "theoretical_loss": 4.589175476195651, + "tokens_seen": 133365760 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048463390170511535, + "loss": 3.1795, + "theoretical_loss": 4.588875581109523, + "tokens_seen": 133431296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048462387161484453, + "loss": 3.2039, + "theoretical_loss": 4.588575874503304, + "tokens_seen": 133496832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846138415245737, + "loss": 3.4852, + "theoretical_loss": 4.588276356166096, + "tokens_seen": 133562368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048460381143430295, + "loss": 3.3087, + "theoretical_loss": 4.587977025887342, + "tokens_seen": 133627904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845937813440321, + "loss": 3.1137, + "theoretical_loss": 4.58767788345682, + "tokens_seen": 133693440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845837512537613, + "loss": 3.2587, + "theoretical_loss": 4.5873789286646485, + "tokens_seen": 133758976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048457372116349044, + "loss": 3.3583, + "theoretical_loss": 4.587080161301283, + "tokens_seen": 133824512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845636910732197, + "loss": 3.2645, + "theoretical_loss": 4.586781581157513, + "tokens_seen": 133890048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048455366098294886, + "loss": 3.289, + "theoretical_loss": 4.58648318802447, + "tokens_seen": 133955584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048454363089267804, + "loss": 2.9687, + "theoretical_loss": 4.586184981693611, + "tokens_seen": 134021120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845336008024072, + "loss": 3.1314, + "theoretical_loss": 4.585886961956737, + "tokens_seen": 134086656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048452357071213645, + "loss": 3.6532, + "theoretical_loss": 4.585589128605976, + "tokens_seen": 134152192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845135406218656, + "loss": 3.1767, + "theoretical_loss": 4.585291481433792, + "tokens_seen": 134217728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845035105315948, + "loss": 3.1699, + "theoretical_loss": 4.584994020232981, + "tokens_seen": 134283264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 84861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.49385142326355, + "objective/train/theoretical_loss": 4.58469674479667, + "objective/train/tokens_used": 154808800, + "theoretical_loss": 4.58469674479667, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048449348044132394, + "loss": 3.3437, + "theoretical_loss": 4.58469674479667, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844834503510532, + "loss": 3.3897, + "theoretical_loss": 4.584399654918316, + "tokens_seen": 134414336 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048447342026078236, + "loss": 3.2379, + "theoretical_loss": 4.584102750391709, + "tokens_seen": 134479872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048446339017051154, + "loss": 3.4963, + "theoretical_loss": 4.583806031010965, + "tokens_seen": 134545408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844533600802407, + "loss": 3.4515, + "theoretical_loss": 4.5835094965705325, + "tokens_seen": 134610944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844433299899699, + "loss": 3.323, + "theoretical_loss": 4.5832131468651855, + "tokens_seen": 134676480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844332998996991, + "loss": 3.0751, + "theoretical_loss": 4.582916981690026, + "tokens_seen": 134742016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844232698094283, + "loss": 3.2973, + "theoretical_loss": 4.582621000840484, + "tokens_seen": 134807552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048441323971915745, + "loss": 3.3443, + "theoretical_loss": 4.582325204112314, + "tokens_seen": 134873088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844032096288867, + "loss": 3.2763, + "theoretical_loss": 4.582029591301595, + "tokens_seen": 134938624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048439317953861586, + "loss": 3.3043, + "theoretical_loss": 4.581734162204734, + "tokens_seen": 135004160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048438314944834504, + "loss": 3.2092, + "theoretical_loss": 4.5814389166184615, + "tokens_seen": 135069696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843731193580742, + "loss": 3.2379, + "theoretical_loss": 4.581143854339828, + "tokens_seen": 135135232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843630892678034, + "loss": 3.2659, + "theoretical_loss": 4.580848975166209, + "tokens_seen": 135200768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843530591775326, + "loss": 3.0079, + "theoretical_loss": 4.580554278895304, + "tokens_seen": 135266304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843430290872618, + "loss": 3.2396, + "theoretical_loss": 4.58025976532513, + "tokens_seen": 135331840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048433299899699095, + "loss": 3.2323, + "theoretical_loss": 4.579965434254028, + "tokens_seen": 135397376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843229689067202, + "loss": 3.0602, + "theoretical_loss": 4.579671285480657, + "tokens_seen": 135462912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843129388164493, + "loss": 3.2404, + "theoretical_loss": 4.579377318803996, + "tokens_seen": 135528448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048430290872617855, + "loss": 3.0519, + "theoretical_loss": 4.579083534023344, + "tokens_seen": 135593984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048429287863590773, + "loss": 3.3365, + "theoretical_loss": 4.578789930938315, + "tokens_seen": 135659520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842828485456369, + "loss": 3.2243, + "theoretical_loss": 4.5784965093488434, + "tokens_seen": 135725056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842728184553661, + "loss": 3.1942, + "theoretical_loss": 4.578203269055179, + "tokens_seen": 135790592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048426278836509527, + "loss": 3.4092, + "theoretical_loss": 4.577910209857889, + "tokens_seen": 135856128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048425275827482445, + "loss": 3.1487, + "theoretical_loss": 4.577617331557854, + "tokens_seen": 135921664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 86086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4546592235565186, + "objective/train/theoretical_loss": 4.577324633956271, + "objective/train/tokens_used": 156447200, + "theoretical_loss": 4.577324633956271, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842427281845537, + "loss": 3.3413, + "theoretical_loss": 4.577324633956271, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048423269809428287, + "loss": 3.4106, + "theoretical_loss": 4.5770321168546495, + "tokens_seen": 136052736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048422266800401205, + "loss": 3.4241, + "theoretical_loss": 4.576739780054815, + "tokens_seen": 136118272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842126379137413, + "loss": 3.254, + "theoretical_loss": 4.576447623358905, + "tokens_seen": 136183808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842026078234704, + "loss": 3.0941, + "theoretical_loss": 4.576155646569367, + "tokens_seen": 136249344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048419257773319965, + "loss": 3.3943, + "theoretical_loss": 4.575863849488963, + "tokens_seen": 136314880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841825476429288, + "loss": 3.2206, + "theoretical_loss": 4.575572231920763, + "tokens_seen": 136380416 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484172517552658, + "loss": 3.1229, + "theoretical_loss": 4.575280793668151, + "tokens_seen": 136445952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841624874623872, + "loss": 2.9849, + "theoretical_loss": 4.574989534534817, + "tokens_seen": 136511488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048415245737211637, + "loss": 3.2635, + "theoretical_loss": 4.574698454324764, + "tokens_seen": 136577024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048414242728184555, + "loss": 3.1675, + "theoretical_loss": 4.574407552842299, + "tokens_seen": 136642560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048413239719157473, + "loss": 3.5906, + "theoretical_loss": 4.57411682989204, + "tokens_seen": 136708096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841223671013039, + "loss": 3.2567, + "theoretical_loss": 4.57382628527891, + "tokens_seen": 136773632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048411233701103315, + "loss": 3.5297, + "theoretical_loss": 4.573535918808141, + "tokens_seen": 136839168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841023069207623, + "loss": 3.3214, + "theoretical_loss": 4.573245730285268, + "tokens_seen": 136904704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840922768304915, + "loss": 3.2527, + "theoretical_loss": 4.572955719516132, + "tokens_seen": 136970240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048408224674022064, + "loss": 3.0917, + "theoretical_loss": 4.572665886306881, + "tokens_seen": 137035776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840722166499499, + "loss": 3.0382, + "theoretical_loss": 4.572376230463965, + "tokens_seen": 137101312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048406218655967906, + "loss": 3.3253, + "theoretical_loss": 4.572086751794135, + "tokens_seen": 137166848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048405215646940824, + "loss": 3.4228, + "theoretical_loss": 4.571797450104451, + "tokens_seen": 137232384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840421263791374, + "loss": 3.4935, + "theoretical_loss": 4.571508325202268, + "tokens_seen": 137297920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048403209628886665, + "loss": 3.2935, + "theoretical_loss": 4.5712193768952485, + "tokens_seen": 137363456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840220661985958, + "loss": 3.0949, + "theoretical_loss": 4.570930604991352, + "tokens_seen": 137428992 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484012036108325, + "loss": 3.0731, + "theoretical_loss": 4.57064200929884, + "tokens_seen": 137494528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048400200601805414, + "loss": 3.2526, + "theoretical_loss": 4.570353589626273, + "tokens_seen": 137560064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 86654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.495647668838501, + "objective/train/theoretical_loss": 4.570065345782512, + "objective/train/tokens_used": 158085600, + "theoretical_loss": 4.570065345782512, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839919759277834, + "loss": 3.3585, + "theoretical_loss": 4.570065345782512, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048398194583751256, + "loss": 3.331, + "theoretical_loss": 4.569777277576713, + "tokens_seen": 137691136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397191574724174, + "loss": 3.2636, + "theoretical_loss": 4.569489384818334, + "tokens_seen": 137756672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839618856569709, + "loss": 3.3772, + "theoretical_loss": 4.569201667317128, + "tokens_seen": 137822208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839518555667001, + "loss": 3.1082, + "theoretical_loss": 4.568914124883145, + "tokens_seen": 137887744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839418254764293, + "loss": 3.567, + "theoretical_loss": 4.568626757326731, + "tokens_seen": 137953280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839317953861585, + "loss": 3.3668, + "theoretical_loss": 4.568339564458528, + "tokens_seen": 138018816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048392176529588765, + "loss": 3.2898, + "theoretical_loss": 4.568052546089472, + "tokens_seen": 138084352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839117352056169, + "loss": 3.2154, + "theoretical_loss": 4.567765702030792, + "tokens_seen": 138149888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048390170511534606, + "loss": 3.3787, + "theoretical_loss": 4.567479032094015, + "tokens_seen": 138215424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048389167502507524, + "loss": 3.2051, + "theoretical_loss": 4.567192536090956, + "tokens_seen": 138280960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838816449348044, + "loss": 3.2675, + "theoretical_loss": 4.566906213833725, + "tokens_seen": 138346496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838716148445336, + "loss": 3.4467, + "theoretical_loss": 4.566620065134725, + "tokens_seen": 138412032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838615847542628, + "loss": 3.3219, + "theoretical_loss": 4.566334089806647, + "tokens_seen": 138477568 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483851554663992, + "loss": 3.3479, + "theoretical_loss": 4.566048287662475, + "tokens_seen": 138543104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048384152457372115, + "loss": 3.3473, + "theoretical_loss": 4.565762658515482, + "tokens_seen": 138608640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838314944834504, + "loss": 3.3177, + "theoretical_loss": 4.565477202179232, + "tokens_seen": 138674176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838214643931795, + "loss": 3.2072, + "theoretical_loss": 4.565191918467576, + "tokens_seen": 138739712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381143430290875, + "loss": 3.4187, + "theoretical_loss": 4.564906807194655, + "tokens_seen": 138805248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048380140421263793, + "loss": 2.8605, + "theoretical_loss": 4.564621868174897, + "tokens_seen": 138870784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837913741223671, + "loss": 3.0615, + "theoretical_loss": 4.564337101223017, + "tokens_seen": 138936320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837813440320963, + "loss": 3.3648, + "theoretical_loss": 4.564052506154015, + "tokens_seen": 139001856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377131394182547, + "loss": 3.051, + "theoretical_loss": 4.563768082783182, + "tokens_seen": 139067392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048376128385155465, + "loss": 3.1675, + "theoretical_loss": 4.563483830926089, + "tokens_seen": 139132928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837512537612839, + "loss": 3.1113, + "theoretical_loss": 4.563199750398592, + "tokens_seen": 139198464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 87912, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.470696210861206, + "objective/train/theoretical_loss": 4.562915841016838, + "objective/train/tokens_used": 159724000, + "theoretical_loss": 4.562915841016838, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483741223671013, + "loss": 3.5703, + "theoretical_loss": 4.562915841016838, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373119358074225, + "loss": 3.2538, + "theoretical_loss": 4.5626321025972505, + "tokens_seen": 139329536 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048372116349047143, + "loss": 3.2385, + "theoretical_loss": 4.562348534956538, + "tokens_seen": 139395072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837111334002006, + "loss": 3.4436, + "theoretical_loss": 4.562065137911691, + "tokens_seen": 139460608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837011033099298, + "loss": 3.1403, + "theoretical_loss": 4.561781911279987, + "tokens_seen": 139526144 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483691073219659, + "loss": 3.1669, + "theoretical_loss": 4.5614988548789785, + "tokens_seen": 139591680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048368104312938816, + "loss": 3.1656, + "theoretical_loss": 4.561215968526502, + "tokens_seen": 139657216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836710130391174, + "loss": 3.2068, + "theoretical_loss": 4.560933252040675, + "tokens_seen": 139722752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836609829488465, + "loss": 3.2876, + "theoretical_loss": 4.56065070523989, + "tokens_seen": 139788288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048365095285857575, + "loss": 3.7268, + "theoretical_loss": 4.560368327942827, + "tokens_seen": 139853824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836409227683049, + "loss": 3.0936, + "theoretical_loss": 4.560086119968435, + "tokens_seen": 139919360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836308926780341, + "loss": 3.362, + "theoretical_loss": 4.559804081135949, + "tokens_seen": 139984896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836208625877633, + "loss": 3.4023, + "theoretical_loss": 4.559522211264877, + "tokens_seen": 140050432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836108324974925, + "loss": 3.299, + "theoretical_loss": 4.559240510175006, + "tokens_seen": 140115968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048360080240722166, + "loss": 3.5194, + "theoretical_loss": 4.558958977686398, + "tokens_seen": 140181504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359077231695084, + "loss": 3.1333, + "theoretical_loss": 4.558677613619393, + "tokens_seen": 140247040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048358074222668, + "loss": 3.1792, + "theoretical_loss": 4.558396417794603, + "tokens_seen": 140312576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357071213640926, + "loss": 3.2967, + "theoretical_loss": 4.558115390032916, + "tokens_seen": 140378112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835606820461384, + "loss": 3.2453, + "theoretical_loss": 4.557834530155497, + "tokens_seen": 140443648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835506519558676, + "loss": 3.2309, + "theoretical_loss": 4.5575538379837806, + "tokens_seen": 140509184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835406218655968, + "loss": 3.1676, + "theoretical_loss": 4.557273313339477, + "tokens_seen": 140574720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483530591775326, + "loss": 3.3045, + "theoretical_loss": 4.556992956044569, + "tokens_seen": 140640256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048352056168505516, + "loss": 3.4098, + "theoretical_loss": 4.55671276592131, + "tokens_seen": 140705792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048351053159478434, + "loss": 3.2187, + "theoretical_loss": 4.556432742792225, + "tokens_seen": 140771328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835005015045135, + "loss": 3.2732, + "theoretical_loss": 4.55615288648011, + "tokens_seen": 140836864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 88907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.168332576751709, + "objective/train/theoretical_loss": 4.555873196808033, + "objective/train/tokens_used": 161362400, + "theoretical_loss": 4.555873196808033, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349047141424276, + "loss": 3.2827, + "theoretical_loss": 4.555873196808033, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048348044132397194, + "loss": 3.0851, + "theoretical_loss": 4.555593673599329, + "tokens_seen": 140967936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834704112337011, + "loss": 3.2459, + "theoretical_loss": 4.555314316677606, + "tokens_seen": 141033472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834603811434303, + "loss": 3.1295, + "theoretical_loss": 4.555035125866738, + "tokens_seen": 141099008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834503510531595, + "loss": 3.2718, + "theoretical_loss": 4.5547561009908675, + "tokens_seen": 141164544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834403209628887, + "loss": 3.4012, + "theoretical_loss": 4.554477241874405, + "tokens_seen": 141230080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343029087261785, + "loss": 3.2815, + "theoretical_loss": 4.554198548342029, + "tokens_seen": 141295616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834202607823471, + "loss": 3.3752, + "theoretical_loss": 4.553920020218683, + "tokens_seen": 141361152 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048341023069207626, + "loss": 3.4304, + "theoretical_loss": 4.55364165732958, + "tokens_seen": 141426688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048340020060180544, + "loss": 3.06, + "theoretical_loss": 4.553363459500193, + "tokens_seen": 141492224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833901705115346, + "loss": 3.4717, + "theoretical_loss": 4.553085426556265, + "tokens_seen": 141557760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833801404212638, + "loss": 3.2863, + "theoretical_loss": 4.552807558323801, + "tokens_seen": 141623296 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483370110330993, + "loss": 3.5702, + "theoretical_loss": 4.552529854629072, + "tokens_seen": 141688832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833600802407222, + "loss": 3.3102, + "theoretical_loss": 4.552252315298611, + "tokens_seen": 141754368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048335005015045135, + "loss": 3.2317, + "theoretical_loss": 4.551974940159212, + "tokens_seen": 141819904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833400200601806, + "loss": 3.3219, + "theoretical_loss": 4.551697729037938, + "tokens_seen": 141885440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833299899699097, + "loss": 3.4492, + "theoretical_loss": 4.551420681762107, + "tokens_seen": 141950976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048331995987963895, + "loss": 3.2021, + "theoretical_loss": 4.551143798159299, + "tokens_seen": 142016512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048330992978936813, + "loss": 3.2159, + "theoretical_loss": 4.550867078057361, + "tokens_seen": 142082048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832998996990973, + "loss": 3.2044, + "theoretical_loss": 4.550590521284393, + "tokens_seen": 142147584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832898696088265, + "loss": 3.1647, + "theoretical_loss": 4.550314127668761, + "tokens_seen": 142213120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048327983951855567, + "loss": 3.4173, + "theoretical_loss": 4.550037897039086, + "tokens_seen": 142278656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048326980942828485, + "loss": 3.0888, + "theoretical_loss": 4.54976182922425, + "tokens_seen": 142344192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832597793380141, + "loss": 3.0527, + "theoretical_loss": 4.549485924053392, + "tokens_seen": 142409728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832497492477432, + "loss": 3.3659, + "theoretical_loss": 4.54921018135591, + "tokens_seen": 142475264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 89401, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.601848602294922, + "objective/train/theoretical_loss": 4.548934600961459, + "objective/train/tokens_used": 163000800, + "theoretical_loss": 4.548934600961459, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048323971915747245, + "loss": 3.4687, + "theoretical_loss": 4.548934600961459, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048322968906720163, + "loss": 3.2753, + "theoretical_loss": 4.54865918269995, + "tokens_seen": 142606336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832196589769308, + "loss": 3.1228, + "theoretical_loss": 4.548383926401552, + "tokens_seen": 142671872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048320962888666, + "loss": 3.0876, + "theoretical_loss": 4.548108831896688, + "tokens_seen": 142737408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831995987963892, + "loss": 3.44, + "theoretical_loss": 4.547833899016037, + "tokens_seen": 142802944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048318956870611836, + "loss": 3.0777, + "theoretical_loss": 4.547559127590535, + "tokens_seen": 142868480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831795386158476, + "loss": 3.5091, + "theoretical_loss": 4.547284517451367, + "tokens_seen": 142934016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831695085255767, + "loss": 3.3175, + "theoretical_loss": 4.547010068429978, + "tokens_seen": 142999552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048315947843530595, + "loss": 3.1291, + "theoretical_loss": 4.546735780358061, + "tokens_seen": 143065088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831494483450351, + "loss": 3.1034, + "theoretical_loss": 4.546461653067565, + "tokens_seen": 143130624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831394182547643, + "loss": 3.0849, + "theoretical_loss": 4.54618768639069, + "tokens_seen": 143196160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831293881644935, + "loss": 3.0665, + "theoretical_loss": 4.545913880159889, + "tokens_seen": 143261696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831193580742227, + "loss": 3.2806, + "theoretical_loss": 4.545640234207866, + "tokens_seen": 143327232 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048310932798395186, + "loss": 3.3249, + "theoretical_loss": 4.545366748367574, + "tokens_seen": 143392768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048309929789368104, + "loss": 3.4583, + "theoretical_loss": 4.545093422472218, + "tokens_seen": 143458304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830892678034102, + "loss": 3.5278, + "theoretical_loss": 4.544820256355253, + "tokens_seen": 143523840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048307923771313946, + "loss": 3.0959, + "theoretical_loss": 4.544547249850384, + "tokens_seen": 143589376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830692076228686, + "loss": 3.4636, + "theoretical_loss": 4.544274402791562, + "tokens_seen": 143654912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830591775325978, + "loss": 3.1348, + "theoretical_loss": 4.544001715012989, + "tokens_seen": 143720448 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483049147442327, + "loss": 3.3713, + "theoretical_loss": 4.543729186349115, + "tokens_seen": 143785984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830391173520562, + "loss": 3.334, + "theoretical_loss": 4.5434568166346345, + "tokens_seen": 143851520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048302908726178536, + "loss": 3.2556, + "theoretical_loss": 4.543184605704493, + "tokens_seen": 143917056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048301905717151454, + "loss": 3.24, + "theoretical_loss": 4.542912553393879, + "tokens_seen": 143982592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830090270812437, + "loss": 3.0543, + "theoretical_loss": 4.542640659538229, + "tokens_seen": 144048128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048299899699097296, + "loss": 3.3306, + "theoretical_loss": 4.542368923973226, + "tokens_seen": 144113664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 90537, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.076249361038208, + "objective/train/theoretical_loss": 4.542097346534794, + "objective/train/tokens_used": 164639200, + "theoretical_loss": 4.542097346534794, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829889669007021, + "loss": 3.2985, + "theoretical_loss": 4.542097346534794, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829789368104313, + "loss": 3.2073, + "theoretical_loss": 4.541825927059106, + "tokens_seen": 144244736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048296890672016045, + "loss": 3.3592, + "theoretical_loss": 4.541554665382575, + "tokens_seen": 144310272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829588766298897, + "loss": 3.5993, + "theoretical_loss": 4.541283561341862, + "tokens_seen": 144375808 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294884653961887, + "loss": 3.0916, + "theoretical_loss": 4.541012614773868, + "tokens_seen": 144441344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048293881644934805, + "loss": 3.2114, + "theoretical_loss": 4.540741825515736, + "tokens_seen": 144506880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292878635907723, + "loss": 3.4082, + "theoretical_loss": 4.540471193404855, + "tokens_seen": 144572416 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048291875626880646, + "loss": 3.1695, + "theoretical_loss": 4.54020071827885, + "tokens_seen": 144637952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829087261785356, + "loss": 3.4008, + "theoretical_loss": 4.539930399975596, + "tokens_seen": 144703488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828986960882648, + "loss": 3.1431, + "theoretical_loss": 4.539660238333198, + "tokens_seen": 144769024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048288866599799395, + "loss": 3.3694, + "theoretical_loss": 4.53939023319001, + "tokens_seen": 144834560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828786359077232, + "loss": 3.3767, + "theoretical_loss": 4.539120384384621, + "tokens_seen": 144900096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048286860581745237, + "loss": 3.5769, + "theoretical_loss": 4.538850691755861, + "tokens_seen": 144965632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048285857572718155, + "loss": 3.3304, + "theoretical_loss": 4.538581155142799, + "tokens_seen": 145031168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048284854563691073, + "loss": 3.1185, + "theoretical_loss": 4.538311774384743, + "tokens_seen": 145096704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828385155466399, + "loss": 3.4686, + "theoretical_loss": 4.538042549321238, + "tokens_seen": 145162240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828284854563691, + "loss": 3.1808, + "theoretical_loss": 4.537773479792067, + "tokens_seen": 145227776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048281845536609833, + "loss": 3.2259, + "theoretical_loss": 4.537504565637248, + "tokens_seen": 145293312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048280842527582746, + "loss": 3.3492, + "theoretical_loss": 4.537235806697041, + "tokens_seen": 145358848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827983951855567, + "loss": 3.4474, + "theoretical_loss": 4.536967202811935, + "tokens_seen": 145424384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827883650952858, + "loss": 3.2921, + "theoretical_loss": 4.536698753822661, + "tokens_seen": 145489920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048277833500501505, + "loss": 3.4976, + "theoretical_loss": 4.53643045957018, + "tokens_seen": 145555456 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048276830491474423, + "loss": 3.4497, + "theoretical_loss": 4.536162319895695, + "tokens_seen": 145620992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827582748244734, + "loss": 3.4119, + "theoretical_loss": 4.535894334640634, + "tokens_seen": 145686528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827482447342026, + "loss": 3.0995, + "theoretical_loss": 4.535626503646666, + "tokens_seen": 145752064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 91176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9223389625549316, + "objective/train/theoretical_loss": 4.535358826755691, + "objective/train/tokens_used": 166277600, + "theoretical_loss": 4.535358826755691, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048273821464393183, + "loss": 3.2178, + "theoretical_loss": 4.535358826755691, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482728184553661, + "loss": 3.4704, + "theoretical_loss": 4.535091303809842, + "tokens_seen": 145883136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827181544633902, + "loss": 3.2936, + "theoretical_loss": 4.534823934651484, + "tokens_seen": 145948672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827081243731194, + "loss": 3.1424, + "theoretical_loss": 4.5345567191232155, + "tokens_seen": 146014208 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048269809428284856, + "loss": 3.3168, + "theoretical_loss": 4.534289657067865, + "tokens_seen": 146079744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826880641925778, + "loss": 2.9011, + "theoretical_loss": 4.5340227483284945, + "tokens_seen": 146145280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826780341023069, + "loss": 3.2725, + "theoretical_loss": 4.533755992748395, + "tokens_seen": 146210816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266800401203615, + "loss": 3.2624, + "theoretical_loss": 4.533489390171088, + "tokens_seen": 146276352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826579739217653, + "loss": 3.4202, + "theoretical_loss": 4.533222940440325, + "tokens_seen": 146341888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826479438314945, + "loss": 3.1386, + "theoretical_loss": 4.532956643400087, + "tokens_seen": 146407424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826379137412237, + "loss": 3.3923, + "theoretical_loss": 4.532690498894585, + "tokens_seen": 146472960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826278836509529, + "loss": 3.4023, + "theoretical_loss": 4.5324245067682565, + "tokens_seen": 146538496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048261785356068206, + "loss": 3.1156, + "theoretical_loss": 4.532158666865769, + "tokens_seen": 146604032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048260782347041124, + "loss": 3.1851, + "theoretical_loss": 4.5318929790320155, + "tokens_seen": 146669568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825977933801404, + "loss": 3.0943, + "theoretical_loss": 4.53162744311212, + "tokens_seen": 146735104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258776328986966, + "loss": 3.392, + "theoretical_loss": 4.53136205895143, + "tokens_seen": 146800640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825777331995988, + "loss": 3.4047, + "theoretical_loss": 4.531096826395519, + "tokens_seen": 146866176 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482567703109328, + "loss": 3.2556, + "theoretical_loss": 4.5308317452901905, + "tokens_seen": 146931712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825576730190572, + "loss": 3.1769, + "theoretical_loss": 4.530566815481469, + "tokens_seen": 146997248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825476429287864, + "loss": 3.2533, + "theoretical_loss": 4.530302036815607, + "tokens_seen": 147062784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048253761283851556, + "loss": 3.3405, + "theoretical_loss": 4.53003740913908, + "tokens_seen": 147128320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252758274824474, + "loss": 3.3032, + "theoretical_loss": 4.529772932298588, + "tokens_seen": 147193856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825175526579739, + "loss": 3.4738, + "theoretical_loss": 4.529508606141057, + "tokens_seen": 147259392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048250752256770316, + "loss": 3.2304, + "theoretical_loss": 4.529244430513632, + "tokens_seen": 147324928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824974924774323, + "loss": 3.2967, + "theoretical_loss": 4.528980405263686, + "tokens_seen": 147390464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 92532, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4305601119995117, + "objective/train/theoretical_loss": 4.528716530238812, + "objective/train/tokens_used": 167916000, + "theoretical_loss": 4.528716530238812, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824874623871615, + "loss": 3.4659, + "theoretical_loss": 4.528716530238812, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048247743229689065, + "loss": 3.2645, + "theoretical_loss": 4.528452805286824, + "tokens_seen": 147521536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824674022066199, + "loss": 3.2269, + "theoretical_loss": 4.528189230255759, + "tokens_seen": 147587072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048245737211634907, + "loss": 3.2869, + "theoretical_loss": 4.527925804993875, + "tokens_seen": 147652608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244734202607825, + "loss": 3.3874, + "theoretical_loss": 4.5276625293496515, + "tokens_seen": 147718144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048243731193580743, + "loss": 3.1961, + "theoretical_loss": 4.527399403171788, + "tokens_seen": 147783680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048242728184553666, + "loss": 3.3541, + "theoretical_loss": 4.5271364263092035, + "tokens_seen": 147849216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824172517552658, + "loss": 3.2284, + "theoretical_loss": 4.526873598611038, + "tokens_seen": 147914752 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482407221664995, + "loss": 3.1859, + "theoretical_loss": 4.52661091992665, + "tokens_seen": 147980288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048239719157472415, + "loss": 3.3306, + "theoretical_loss": 4.526348390105614, + "tokens_seen": 148045824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823871614844534, + "loss": 3.0245, + "theoretical_loss": 4.526086008997728, + "tokens_seen": 148111360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048237713139418257, + "loss": 3.1419, + "theoretical_loss": 4.525823776453004, + "tokens_seen": 148176896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048236710130391175, + "loss": 3.0672, + "theoretical_loss": 4.525561692321673, + "tokens_seen": 148242432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048235707121364093, + "loss": 3.259, + "theoretical_loss": 4.525299756454182, + "tokens_seen": 148307968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823470411233701, + "loss": 3.3995, + "theoretical_loss": 4.525037968701198, + "tokens_seen": 148373504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823370110330993, + "loss": 3.1846, + "theoretical_loss": 4.5247763289136, + "tokens_seen": 148439040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048232698094282853, + "loss": 3.3233, + "theoretical_loss": 4.524514836942485, + "tokens_seen": 148504576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048231695085255766, + "loss": 3.1244, + "theoretical_loss": 4.524253492639165, + "tokens_seen": 148570112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823069207622869, + "loss": 3.1294, + "theoretical_loss": 4.523992295855169, + "tokens_seen": 148635648 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482296890672016, + "loss": 3.4438, + "theoretical_loss": 4.523731246442237, + "tokens_seen": 148701184 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048228686058174525, + "loss": 3.2735, + "theoretical_loss": 4.523470344252326, + "tokens_seen": 148766720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048227683049147443, + "loss": 3.2357, + "theoretical_loss": 4.523209589137606, + "tokens_seen": 148832256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822668004012036, + "loss": 3.3941, + "theoretical_loss": 4.522948980950461, + "tokens_seen": 148897792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822567703109328, + "loss": 3.0532, + "theoretical_loss": 4.522688519543487, + "tokens_seen": 148963328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224674022066203, + "loss": 3.1432, + "theoretical_loss": 4.5224282047694935, + "tokens_seen": 149028864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 93022, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5269510746002197, + "objective/train/theoretical_loss": 4.522168036481502, + "objective/train/tokens_used": 169554400, + "theoretical_loss": 4.522168036481502, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048223671013039116, + "loss": 3.5101, + "theoretical_loss": 4.522168036481502, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822266800401204, + "loss": 3.2561, + "theoretical_loss": 4.521908014532746, + "tokens_seen": 149159936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822166499498495, + "loss": 3.3344, + "theoretical_loss": 4.52164813877667, + "tokens_seen": 149225472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048220661985957876, + "loss": 3.2209, + "theoretical_loss": 4.52138840906693, + "tokens_seen": 149291008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048219658976930794, + "loss": 3.1369, + "theoretical_loss": 4.521128825257392, + "tokens_seen": 149356544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821865596790371, + "loss": 3.1751, + "theoretical_loss": 4.520869387202134, + "tokens_seen": 149422080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821765295887663, + "loss": 2.9797, + "theoretical_loss": 4.52061009475544, + "tokens_seen": 149487616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821664994984955, + "loss": 3.196, + "theoretical_loss": 4.52035094777181, + "tokens_seen": 149553152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048215646940822466, + "loss": 3.3068, + "theoretical_loss": 4.520091946105946, + "tokens_seen": 149618688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821464393179539, + "loss": 3.4288, + "theoretical_loss": 4.519833089612764, + "tokens_seen": 149684224 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482136409227683, + "loss": 3.2192, + "theoretical_loss": 4.519574378147384, + "tokens_seen": 149749760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048212637913741226, + "loss": 3.291, + "theoretical_loss": 4.519315811565137, + "tokens_seen": 149815296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821163490471414, + "loss": 3.3752, + "theoretical_loss": 4.51905738972156, + "tokens_seen": 149880832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821063189568706, + "loss": 3.6035, + "theoretical_loss": 4.518799112472401, + "tokens_seen": 149946368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820962888665998, + "loss": 3.5239, + "theoretical_loss": 4.518540979673606, + "tokens_seen": 150011904 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482086258776329, + "loss": 3.1651, + "theoretical_loss": 4.518282991181337, + "tokens_seen": 150077440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048207622868605816, + "loss": 3.1406, + "theoretical_loss": 4.518025146851956, + "tokens_seen": 150142976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820661985957874, + "loss": 3.2538, + "theoretical_loss": 4.517767446542033, + "tokens_seen": 150208512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048205616850551653, + "loss": 3.1976, + "theoretical_loss": 4.517509890108343, + "tokens_seen": 150274048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204613841524576, + "loss": 3.1097, + "theoretical_loss": 4.5172524774078635, + "tokens_seen": 150339584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820361083249749, + "loss": 3.3081, + "theoretical_loss": 4.5169952082977805, + "tokens_seen": 150405120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820260782347041, + "loss": 3.2403, + "theoretical_loss": 4.51673808263548, + "tokens_seen": 150470656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820160481444333, + "loss": 3.0648, + "theoretical_loss": 4.516481100278555, + "tokens_seen": 150536192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820060180541625, + "loss": 3.1256, + "theoretical_loss": 4.516224261084799, + "tokens_seen": 150601728 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048199598796389167, + "loss": 3.0304, + "theoretical_loss": 4.515967564912211, + "tokens_seen": 150667264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 94173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2401325702667236, + "objective/train/theoretical_loss": 4.515711011618992, + "objective/train/tokens_used": 171192800, + "theoretical_loss": 4.515711011618992, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048198595787362085, + "loss": 3.2269, + "theoretical_loss": 4.515711011618992, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004819759277833501, + "loss": 3.1091, + "theoretical_loss": 4.515454601063542, + "tokens_seen": 150798336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048196589769307927, + "loss": 3.2182, + "theoretical_loss": 4.5151983331044665, + "tokens_seen": 150863872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048195586760280845, + "loss": 3.2107, + "theoretical_loss": 4.514942207600572, + "tokens_seen": 150929408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048194583751253763, + "loss": 3.1827, + "theoretical_loss": 4.514686224410863, + "tokens_seen": 150994944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048193580742226686, + "loss": 3.3203, + "theoretical_loss": 4.514430383394549, + "tokens_seen": 151060480 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481925777331996, + "loss": 3.0874, + "theoretical_loss": 4.514174684411037, + "tokens_seen": 151126016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004819157472417252, + "loss": 3.2352, + "theoretical_loss": 4.513919127319935, + "tokens_seen": 151191552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048190571715145435, + "loss": 3.3179, + "theoretical_loss": 4.513663711981049, + "tokens_seen": 151257088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818956870611836, + "loss": 3.266, + "theoretical_loss": 4.513408438254386, + "tokens_seen": 151322624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048188565697091277, + "loss": 3.1682, + "theoretical_loss": 4.5131533060001505, + "tokens_seen": 151388160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048187562688064195, + "loss": 3.2016, + "theoretical_loss": 4.512898315078747, + "tokens_seen": 151453696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048186559679037113, + "loss": 3.2221, + "theoretical_loss": 4.512643465350775, + "tokens_seen": 151519232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818555667001003, + "loss": 3.394, + "theoretical_loss": 4.512388756677037, + "tokens_seen": 151584768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818455366098295, + "loss": 3.3002, + "theoretical_loss": 4.5121341889185285, + "tokens_seen": 151650304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048183550651955873, + "loss": 3.131, + "theoretical_loss": 4.511879761936442, + "tokens_seen": 151715840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048182547642928786, + "loss": 3.3703, + "theoretical_loss": 4.511625475592169, + "tokens_seen": 151781376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818154463390171, + "loss": 3.2669, + "theoretical_loss": 4.511371329747295, + "tokens_seen": 151846912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818054162487462, + "loss": 3.0424, + "theoretical_loss": 4.511117324263603, + "tokens_seen": 151912448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048179538615847545, + "loss": 3.6565, + "theoretical_loss": 4.5108634590030725, + "tokens_seen": 151977984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048178535606820463, + "loss": 3.411, + "theoretical_loss": 4.510609733827873, + "tokens_seen": 152043520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817753259779338, + "loss": 3.1894, + "theoretical_loss": 4.5103561486003745, + "tokens_seen": 152109056 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481765295887663, + "loss": 3.053, + "theoretical_loss": 4.51010270318314, + "tokens_seen": 152174592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048175526579739223, + "loss": 3.1504, + "theoretical_loss": 4.509849397438925, + "tokens_seen": 152240128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048174523570712136, + "loss": 3.2262, + "theoretical_loss": 4.509596231230679, + "tokens_seen": 152305664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 94867, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.429643392562866, + "objective/train/theoretical_loss": 4.509343204421548, + "objective/train/tokens_used": 172831200, + "theoretical_loss": 4.509343204421548, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817352056168506, + "loss": 3.4209, + "theoretical_loss": 4.509343204421548, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817251755265797, + "loss": 3.2597, + "theoretical_loss": 4.509090316874865, + "tokens_seen": 152436736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048171514543630896, + "loss": 3.2857, + "theoretical_loss": 4.508837568454162, + "tokens_seen": 152502272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048170511534603814, + "loss": 3.5512, + "theoretical_loss": 4.508584959023159, + "tokens_seen": 152567808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816950852557673, + "loss": 3.3878, + "theoretical_loss": 4.50833248844577, + "tokens_seen": 152633344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816850551654965, + "loss": 3.3808, + "theoretical_loss": 4.5080801565861, + "tokens_seen": 152698880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816750250752257, + "loss": 3.1163, + "theoretical_loss": 4.507827963308447, + "tokens_seen": 152764416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048166499498495486, + "loss": 3.2056, + "theoretical_loss": 4.507575908477296, + "tokens_seen": 152829952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816549648946841, + "loss": 3.1235, + "theoretical_loss": 4.507323991957325, + "tokens_seen": 152895488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816449348044132, + "loss": 3.439, + "theoretical_loss": 4.507072213613402, + "tokens_seen": 152961024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048163490471414246, + "loss": 3.2998, + "theoretical_loss": 4.506820573310586, + "tokens_seen": 153026560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816248746238716, + "loss": 3.5354, + "theoretical_loss": 4.506569070914123, + "tokens_seen": 153092096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816148445336008, + "loss": 3.1693, + "theoretical_loss": 4.506317706289451, + "tokens_seen": 153157632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048160481444333, + "loss": 3.3531, + "theoretical_loss": 4.506066479302193, + "tokens_seen": 153223168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815947843530592, + "loss": 3.3859, + "theoretical_loss": 4.505815389818165, + "tokens_seen": 153288704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048158475426278837, + "loss": 3.2548, + "theoretical_loss": 4.505564437703368, + "tokens_seen": 153354240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815747241725176, + "loss": 3.1131, + "theoretical_loss": 4.5053136228239925, + "tokens_seen": 153419776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048156469408224673, + "loss": 3.3206, + "theoretical_loss": 4.505062945046413, + "tokens_seen": 153485312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048155466399197596, + "loss": 3.3988, + "theoretical_loss": 4.504812404237197, + "tokens_seen": 153550848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815446339017051, + "loss": 3.1752, + "theoretical_loss": 4.504562000263093, + "tokens_seen": 153616384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815346038114343, + "loss": 3.1504, + "theoretical_loss": 4.504311732991039, + "tokens_seen": 153681920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815245737211635, + "loss": 3.0018, + "theoretical_loss": 4.504061602288159, + "tokens_seen": 153747456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815145436308927, + "loss": 2.9879, + "theoretical_loss": 4.503811608021761, + "tokens_seen": 153812992 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048150451354062187, + "loss": 3.1938, + "theoretical_loss": 4.50356175005934, + "tokens_seen": 153878528 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048149448345035105, + "loss": 3.5199, + "theoretical_loss": 4.503312028268576, + "tokens_seen": 153944064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 95853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.367169141769409, + "objective/train/theoretical_loss": 4.503062442517334, + "objective/train/tokens_used": 174469600, + "theoretical_loss": 4.503062442517334, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048148445336008023, + "loss": 3.257, + "theoretical_loss": 4.503062442517334, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048147442326980947, + "loss": 3.4149, + "theoretical_loss": 4.502812992673661, + "tokens_seen": 154075136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814643931795386, + "loss": 3.3831, + "theoretical_loss": 4.50256367860579, + "tokens_seen": 154140672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048145436308926783, + "loss": 3.2127, + "theoretical_loss": 4.502314500182138, + "tokens_seen": 154206208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048144433299899696, + "loss": 3.0253, + "theoretical_loss": 4.502065457271304, + "tokens_seen": 154271744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814343029087262, + "loss": 3.522, + "theoretical_loss": 4.501816549742072, + "tokens_seen": 154337280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048142427281845537, + "loss": 3.031, + "theoretical_loss": 4.501567777463406, + "tokens_seen": 154402816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048141424272818455, + "loss": 3.33, + "theoretical_loss": 4.501319140304456, + "tokens_seen": 154468352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048140421263791373, + "loss": 3.0943, + "theoretical_loss": 4.50107063813455, + "tokens_seen": 154533888 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048139418254764297, + "loss": 2.9423, + "theoretical_loss": 4.5008222708232, + "tokens_seen": 154599424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813841524573721, + "loss": 3.2762, + "theoretical_loss": 4.500574038240099, + "tokens_seen": 154664960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048137412236710133, + "loss": 3.1341, + "theoretical_loss": 4.500325940255123, + "tokens_seen": 154730496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048136409227683046, + "loss": 3.323, + "theoretical_loss": 4.500077976738324, + "tokens_seen": 154796032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813540621865597, + "loss": 3.2968, + "theoretical_loss": 4.499830147559939, + "tokens_seen": 154861568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813440320962889, + "loss": 3.3287, + "theoretical_loss": 4.499582452590383, + "tokens_seen": 154927104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048133400200601806, + "loss": 3.4478, + "theoretical_loss": 4.499334891700251, + "tokens_seen": 154992640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048132397191574724, + "loss": 3.1801, + "theoretical_loss": 4.499087464760317, + "tokens_seen": 155058176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813139418254764, + "loss": 3.0457, + "theoretical_loss": 4.498840171641534, + "tokens_seen": 155123712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813039117352056, + "loss": 2.9473, + "theoretical_loss": 4.498593012215036, + "tokens_seen": 155189248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048129388164493483, + "loss": 3.1367, + "theoretical_loss": 4.498345986352133, + "tokens_seen": 155254784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048128385155466396, + "loss": 3.2789, + "theoretical_loss": 4.4980990939243135, + "tokens_seen": 155320320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812738214643932, + "loss": 3.265, + "theoretical_loss": 4.497852334803245, + "tokens_seen": 155385856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812637913741223, + "loss": 3.2818, + "theoretical_loss": 4.497605708860771, + "tokens_seen": 155451392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048125376128385156, + "loss": 3.148, + "theoretical_loss": 4.497359215968914, + "tokens_seen": 155516928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048124373119358074, + "loss": 3.1139, + "theoretical_loss": 4.497112855999871, + "tokens_seen": 155582464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 96994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.577995777130127, + "objective/train/theoretical_loss": 4.496866628826018, + "objective/train/tokens_used": 176108000, + "theoretical_loss": 4.496866628826018, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812337011033099, + "loss": 3.366, + "theoretical_loss": 4.496866628826018, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048122367101303916, + "loss": 3.2348, + "theoretical_loss": 4.496620534319904, + "tokens_seen": 155713536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048121364092276834, + "loss": 3.1337, + "theoretical_loss": 4.496374572354256, + "tokens_seen": 155779072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812036108324975, + "loss": 3.4091, + "theoretical_loss": 4.496128742801979, + "tokens_seen": 155844608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811935807422267, + "loss": 3.3832, + "theoretical_loss": 4.495883045536148, + "tokens_seen": 155910144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811835506519559, + "loss": 3.1527, + "theoretical_loss": 4.495637480430018, + "tokens_seen": 155975680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048117352056168506, + "loss": 3.3667, + "theoretical_loss": 4.495392047357015, + "tokens_seen": 156041216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811634904714143, + "loss": 3.1819, + "theoretical_loss": 4.4951467461907395, + "tokens_seen": 156106752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811534603811434, + "loss": 3.2472, + "theoretical_loss": 4.494901576804969, + "tokens_seen": 156172288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048114343029087266, + "loss": 3.3371, + "theoretical_loss": 4.4946565390736515, + "tokens_seen": 156237824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811334002006018, + "loss": 3.3338, + "theoretical_loss": 4.494411632870911, + "tokens_seen": 156303360 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481123370110331, + "loss": 3.3003, + "theoretical_loss": 4.494166858071041, + "tokens_seen": 156368896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811133400200602, + "loss": 3.0684, + "theoretical_loss": 4.493922214548512, + "tokens_seen": 156434432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811033099297894, + "loss": 3.1874, + "theoretical_loss": 4.493677702177964, + "tokens_seen": 156499968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048109327983951857, + "loss": 3.2209, + "theoretical_loss": 4.493433320834209, + "tokens_seen": 156565504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810832497492478, + "loss": 3.3108, + "theoretical_loss": 4.493189070392235, + "tokens_seen": 156631040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048107321965897693, + "loss": 3.0176, + "theoretical_loss": 4.492944950727196, + "tokens_seen": 156696576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048106318956870616, + "loss": 3.0247, + "theoretical_loss": 4.492700961714419, + "tokens_seen": 156762112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810531594784353, + "loss": 3.1494, + "theoretical_loss": 4.492457103229405, + "tokens_seen": 156827648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810431293881645, + "loss": 3.4656, + "theoretical_loss": 4.492213375147822, + "tokens_seen": 156893184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810330992978937, + "loss": 3.1795, + "theoretical_loss": 4.491969777345509, + "tokens_seen": 156958720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810230692076229, + "loss": 3.2572, + "theoretical_loss": 4.491726309698475, + "tokens_seen": 157024256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048101303911735207, + "loss": 3.074, + "theoretical_loss": 4.491482972082901, + "tokens_seen": 157089792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048100300902708125, + "loss": 3.3692, + "theoretical_loss": 4.491239764375134, + "tokens_seen": 157155328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048099297893681043, + "loss": 3.166, + "theoretical_loss": 4.490996686451692, + "tokens_seen": 157220864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 97532, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0817697048187256, + "objective/train/theoretical_loss": 4.4907537381892615, + "objective/train/tokens_used": 177746400, + "theoretical_loss": 4.4907537381892615, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048098294884653967, + "loss": 3.0902, + "theoretical_loss": 4.4907537381892615, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809729187562688, + "loss": 3.3092, + "theoretical_loss": 4.490510919464697, + "tokens_seen": 157351936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048096288866599803, + "loss": 3.1099, + "theoretical_loss": 4.4902682301550225, + "tokens_seen": 157417472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048095285857572716, + "loss": 2.9646, + "theoretical_loss": 4.4900256701374275, + "tokens_seen": 157483008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809428284854564, + "loss": 3.1576, + "theoretical_loss": 4.489783239289271, + "tokens_seen": 157548544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048093279839518557, + "loss": 3.1089, + "theoretical_loss": 4.48954093748808, + "tokens_seen": 157614080 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048092276830491475, + "loss": 3.4034, + "theoretical_loss": 4.489298764611546, + "tokens_seen": 157679616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048091273821464393, + "loss": 2.9096, + "theoretical_loss": 4.48905672053753, + "tokens_seen": 157745152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048090270812437317, + "loss": 3.1601, + "theoretical_loss": 4.488814805144055, + "tokens_seen": 157810688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808926780341023, + "loss": 3.1648, + "theoretical_loss": 4.4885730183093155, + "tokens_seen": 157876224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048088264794383153, + "loss": 3.3537, + "theoretical_loss": 4.488331359911669, + "tokens_seen": 157941760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048087261785356066, + "loss": 3.1861, + "theoretical_loss": 4.488089829829638, + "tokens_seen": 158007296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808625877632899, + "loss": 3.1146, + "theoretical_loss": 4.487848427941911, + "tokens_seen": 158072832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808525576730191, + "loss": 3.4209, + "theoretical_loss": 4.487607154127343, + "tokens_seen": 158138368 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084252758274826, + "loss": 3.4348, + "theoretical_loss": 4.487366008264949, + "tokens_seen": 158203904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048083249749247744, + "loss": 3.3548, + "theoretical_loss": 4.487124990233915, + "tokens_seen": 158269440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808224674022066, + "loss": 3.2445, + "theoretical_loss": 4.486884099913585, + "tokens_seen": 158334976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808124373119358, + "loss": 3.5389, + "theoretical_loss": 4.486643337183471, + "tokens_seen": 158400512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048080240722166503, + "loss": 2.9913, + "theoretical_loss": 4.486402701923245, + "tokens_seen": 158466048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048079237713139416, + "loss": 3.1903, + "theoretical_loss": 4.486162194012745, + "tokens_seen": 158531584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807823470411234, + "loss": 3.3898, + "theoretical_loss": 4.48592181333197, + "tokens_seen": 158597120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807723169508525, + "loss": 3.134, + "theoretical_loss": 4.485681559761082, + "tokens_seen": 158662656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048076228686058176, + "loss": 3.1577, + "theoretical_loss": 4.485441433180408, + "tokens_seen": 158728192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048075225677031094, + "loss": 3.0772, + "theoretical_loss": 4.485201433470432, + "tokens_seen": 158793728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807422266800401, + "loss": 3.3051, + "theoretical_loss": 4.484961560511804, + "tokens_seen": 158859264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 98333, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.562363386154175, + "objective/train/theoretical_loss": 4.484721814185333, + "objective/train/tokens_used": 179384800, + "theoretical_loss": 4.484721814185333, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807321965897693, + "loss": 3.3407, + "theoretical_loss": 4.484721814185333, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072216649949854, + "loss": 3.439, + "theoretical_loss": 4.48448219437199, + "tokens_seen": 158990336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048071213640922766, + "loss": 3.5068, + "theoretical_loss": 4.484242700952908, + "tokens_seen": 159055872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807021063189569, + "loss": 3.4014, + "theoretical_loss": 4.48400333380938, + "tokens_seen": 159121408 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480692076228686, + "loss": 3.3618, + "theoretical_loss": 4.483764092822857, + "tokens_seen": 159186944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068204613841526, + "loss": 3.2863, + "theoretical_loss": 4.483524977874953, + "tokens_seen": 159252480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048067201604814444, + "loss": 3.2132, + "theoretical_loss": 4.48328598884744, + "tokens_seen": 159318016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806619859578736, + "loss": 3.2951, + "theoretical_loss": 4.483047125622251, + "tokens_seen": 159383552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806519558676028, + "loss": 3.3459, + "theoretical_loss": 4.482808388081477, + "tokens_seen": 159449088 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480641925777332, + "loss": 3.1496, + "theoretical_loss": 4.482569776107368, + "tokens_seen": 159514624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048063189568706117, + "loss": 3.2453, + "theoretical_loss": 4.482331289582334, + "tokens_seen": 159580160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806218655967904, + "loss": 3.1819, + "theoretical_loss": 4.482092928388941, + "tokens_seen": 159645696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048061183550651953, + "loss": 3.2609, + "theoretical_loss": 4.4818546924099145, + "tokens_seen": 159711232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048060180541624877, + "loss": 3.2747, + "theoretical_loss": 4.481616581528138, + "tokens_seen": 159776768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048059177532597795, + "loss": 3.1228, + "theoretical_loss": 4.481378595626652, + "tokens_seen": 159842304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058174523570713, + "loss": 3.0437, + "theoretical_loss": 4.481140734588655, + "tokens_seen": 159907840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805717151454363, + "loss": 3.3693, + "theoretical_loss": 4.4809029982975, + "tokens_seen": 159973376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805616850551655, + "loss": 3.4456, + "theoretical_loss": 4.480665386636701, + "tokens_seen": 160038912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048055165496489467, + "loss": 3.1342, + "theoretical_loss": 4.480427899489925, + "tokens_seen": 160104448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805416248746239, + "loss": 2.9679, + "theoretical_loss": 4.480190536740995, + "tokens_seen": 160169984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048053159478435303, + "loss": 3.4332, + "theoretical_loss": 4.479953298273893, + "tokens_seen": 160235520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052156469408227, + "loss": 3.2521, + "theoretical_loss": 4.479716183972753, + "tokens_seen": 160301056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805115346038114, + "loss": 3.5047, + "theoretical_loss": 4.4794791937218665, + "tokens_seen": 160366592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048050150451354063, + "loss": 3.3783, + "theoretical_loss": 4.47924232740568, + "tokens_seen": 160432128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804914744232698, + "loss": 3.3839, + "theoretical_loss": 4.479005584908793, + "tokens_seen": 160497664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 99019, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4908287525177, + "objective/train/theoretical_loss": 4.478768966115963, + "objective/train/tokens_used": 181023200, + "theoretical_loss": 4.478768966115963, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480481444332999, + "loss": 3.1208, + "theoretical_loss": 4.478768966115963, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048047141424272823, + "loss": 3.2379, + "theoretical_loss": 4.478532470912096, + "tokens_seen": 160628736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048046138415245736, + "loss": 3.0995, + "theoretical_loss": 4.478296099182261, + "tokens_seen": 160694272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804513540621866, + "loss": 3.2414, + "theoretical_loss": 4.47805985081167, + "tokens_seen": 160759808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048044132397191577, + "loss": 3.1472, + "theoretical_loss": 4.477823725685695, + "tokens_seen": 160825344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048043129388164495, + "loss": 3.2584, + "theoretical_loss": 4.477587723689862, + "tokens_seen": 160890880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042126379137413, + "loss": 3.2151, + "theoretical_loss": 4.477351844709846, + "tokens_seen": 160956416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048041123370110337, + "loss": 3.464, + "theoretical_loss": 4.477116088631476, + "tokens_seen": 161021952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804012036108325, + "loss": 3.2482, + "theoretical_loss": 4.476880455340735, + "tokens_seen": 161087488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048039117352056173, + "loss": 3.347, + "theoretical_loss": 4.4766449447237555, + "tokens_seen": 161153024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038114343029086, + "loss": 3.2773, + "theoretical_loss": 4.476409556666825, + "tokens_seen": 161218560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803711133400201, + "loss": 3.0332, + "theoretical_loss": 4.476174291056379, + "tokens_seen": 161284096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803610832497493, + "loss": 3.2546, + "theoretical_loss": 4.475939147779008, + "tokens_seen": 161349632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048035105315947846, + "loss": 3.0797, + "theoretical_loss": 4.47570412672145, + "tokens_seen": 161415168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034102306920764, + "loss": 3.1193, + "theoretical_loss": 4.475469227770596, + "tokens_seen": 161480704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803309929789368, + "loss": 3.222, + "theoretical_loss": 4.475234450813487, + "tokens_seen": 161546240 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480320962888666, + "loss": 3.3821, + "theoretical_loss": 4.474999795737315, + "tokens_seen": 161611776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048031093279839523, + "loss": 3.17, + "theoretical_loss": 4.47476526242942, + "tokens_seen": 161677312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048030090270812436, + "loss": 3.0239, + "theoretical_loss": 4.474530850777294, + "tokens_seen": 161742848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802908726178536, + "loss": 3.2871, + "theoretical_loss": 4.474296560668577, + "tokens_seen": 161808384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802808425275827, + "loss": 3.0498, + "theoretical_loss": 4.474062391991058, + "tokens_seen": 161873920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048027081243731196, + "loss": 3.0457, + "theoretical_loss": 4.473828344632677, + "tokens_seen": 161939456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048026078234704114, + "loss": 3.114, + "theoretical_loss": 4.473594418481522, + "tokens_seen": 162004992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802507522567703, + "loss": 3.3295, + "theoretical_loss": 4.473360613425827, + "tokens_seen": 162070528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802407221664995, + "loss": 3.046, + "theoretical_loss": 4.473126929353977, + "tokens_seen": 162136064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 100000, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.690746545791626, + "objective/train/theoretical_loss": 4.472893366154503, + "objective/train/tokens_used": 182661600, + "theoretical_loss": 4.472893366154503, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048023069207622874, + "loss": 2.9009, + "theoretical_loss": 4.472893366154503, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048022066198595786, + "loss": 3.0755, + "theoretical_loss": 4.472659923716087, + "tokens_seen": 162267136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802106318956871, + "loss": 3.0381, + "theoretical_loss": 4.472426601927554, + "tokens_seen": 162332672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020060180541623, + "loss": 3.0383, + "theoretical_loss": 4.472193400677878, + "tokens_seen": 162398208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048019057171514546, + "loss": 3.207, + "theoretical_loss": 4.471960319856182, + "tokens_seen": 162463744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048018054162487464, + "loss": 3.0581, + "theoretical_loss": 4.471727359351734, + "tokens_seen": 162529280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801705115346038, + "loss": 3.323, + "theoretical_loss": 4.4714945190539455, + "tokens_seen": 162594816 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480160481444333, + "loss": 3.3514, + "theoretical_loss": 4.47126179885238, + "tokens_seen": 162660352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801504513540622, + "loss": 3.0515, + "theoretical_loss": 4.471029198636741, + "tokens_seen": 162725888 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014042126379137, + "loss": 2.9545, + "theoretical_loss": 4.470796718296884, + "tokens_seen": 162791424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801303911735206, + "loss": 3.0233, + "theoretical_loss": 4.470564357722804, + "tokens_seen": 162856960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048012036108324973, + "loss": 3.258, + "theoretical_loss": 4.470332116804644, + "tokens_seen": 162922496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048011033099297897, + "loss": 3.2388, + "theoretical_loss": 4.470099995432692, + "tokens_seen": 162988032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048010030090270815, + "loss": 3.0379, + "theoretical_loss": 4.469867993497379, + "tokens_seen": 163053568 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048009027081243733, + "loss": 3.0891, + "theoretical_loss": 4.469636110889283, + "tokens_seen": 163119104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800802407221665, + "loss": 3.2988, + "theoretical_loss": 4.4694043474991245, + "tokens_seen": 163184640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800702106318957, + "loss": 3.0297, + "theoretical_loss": 4.469172703217767, + "tokens_seen": 163250176 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048006018054162487, + "loss": 3.0857, + "theoretical_loss": 4.46894117793622, + "tokens_seen": 163315712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800501504513541, + "loss": 3.1744, + "theoretical_loss": 4.468709771545634, + "tokens_seen": 163381248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004012036108323, + "loss": 3.0103, + "theoretical_loss": 4.4684784839373055, + "tokens_seen": 163446784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048003009027081247, + "loss": 3.2331, + "theoretical_loss": 4.468247315002671, + "tokens_seen": 163512320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800200601805416, + "loss": 3.1392, + "theoretical_loss": 4.468016264633311, + "tokens_seen": 163577856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048001003009027083, + "loss": 3.1491, + "theoretical_loss": 4.4677853327209505, + "tokens_seen": 163643392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 3.3932, + "theoretical_loss": 4.467554519157452, + "tokens_seen": 163708928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799899699097292, + "loss": 3.0553, + "theoretical_loss": 4.467323823834824, + "tokens_seen": 163774464 + }, + { + "debugging/Self-BLEU-5": 0.4946583130096161, + "debugging/distinct-1-grams": 0.815790683224271, + "debugging/distinct-2-grams": 0.9751385384195338, + "debugging/entropy-1-grams": 5.884314010514224, + "debugging/entropy-2-grams": 6.564196933680893, + "debugging/length": 580.8, + "debugging/num_segments": 10, + "epoch": 0.05, + "objective/train/docs_used": 101353, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.959970474243164, + "objective/train/theoretical_loss": 4.467093246645215, + "objective/train/tokens_used": 184300000, + "theoretical_loss": 4.467093246645215, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799799398194584, + "loss": 3.3758, + "theoretical_loss": 4.467093246645215, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047996990972918756, + "loss": 3.0913, + "theoretical_loss": 4.466862787480915, + "tokens_seen": 163905536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047995987963891674, + "loss": 3.201, + "theoretical_loss": 4.466632446234355, + "tokens_seen": 163971072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047994984954864597, + "loss": 3.1657, + "theoretical_loss": 4.466402222798108, + "tokens_seen": 164036608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799398194583751, + "loss": 3.0971, + "theoretical_loss": 4.466172117064889, + "tokens_seen": 164102144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047992978936810433, + "loss": 3.0109, + "theoretical_loss": 4.465942128927549, + "tokens_seen": 164167680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799197592778335, + "loss": 3.2315, + "theoretical_loss": 4.4657122582790825, + "tokens_seen": 164233216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799097291875627, + "loss": 3.2653, + "theoretical_loss": 4.465482505012624, + "tokens_seen": 164298752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798996990972919, + "loss": 3.359, + "theoretical_loss": 4.465252869021448, + "tokens_seen": 164364288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047988966900702106, + "loss": 3.0349, + "theoretical_loss": 4.465023350198967, + "tokens_seen": 164429824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047987963891675024, + "loss": 3.4322, + "theoretical_loss": 4.464793948438733, + "tokens_seen": 164495360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798696088264795, + "loss": 3.1553, + "theoretical_loss": 4.464564663634439, + "tokens_seen": 164560896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798595787362086, + "loss": 3.3483, + "theoretical_loss": 4.464335495679914, + "tokens_seen": 164626432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047984954864593784, + "loss": 3.2043, + "theoretical_loss": 4.464106444469129, + "tokens_seen": 164691968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983951855566696, + "loss": 3.1635, + "theoretical_loss": 4.463877509896189, + "tokens_seen": 164757504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798294884653962, + "loss": 3.298, + "theoretical_loss": 4.463648691855342, + "tokens_seen": 164823040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798194583751254, + "loss": 3.3382, + "theoretical_loss": 4.46341999024097, + "tokens_seen": 164888576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047980942828485456, + "loss": 3.3223, + "theoretical_loss": 4.463191404947595, + "tokens_seen": 164954112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047979939819458374, + "loss": 3.2243, + "theoretical_loss": 4.462962935869875, + "tokens_seen": 165019648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797893681043129, + "loss": 3.1862, + "theoretical_loss": 4.4627345829026055, + "tokens_seen": 165085184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797793380140421, + "loss": 3.2557, + "theoretical_loss": 4.462506345940719, + "tokens_seen": 165150720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047976930792377134, + "loss": 2.9256, + "theoretical_loss": 4.462278224879286, + "tokens_seen": 165216256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975927783350047, + "loss": 2.9158, + "theoretical_loss": 4.462050219613511, + "tokens_seen": 165281792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797492477432297, + "loss": 3.2504, + "theoretical_loss": 4.461822330038737, + "tokens_seen": 165347328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797392176529589, + "loss": 3.042, + "theoretical_loss": 4.461594556050444, + "tokens_seen": 165412864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 101896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.066077470779419, + "objective/train/theoretical_loss": 4.4613668975442415, + "objective/train/tokens_used": 185938400, + "theoretical_loss": 4.4613668975442415, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047972918756268807, + "loss": 3.1283, + "theoretical_loss": 4.4613668975442415, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797191574724173, + "loss": 3.3684, + "theoretical_loss": 4.4611393544158835, + "tokens_seen": 165543936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047970912738214643, + "loss": 3.3009, + "theoretical_loss": 4.460911926561252, + "tokens_seen": 165609472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047969909729187566, + "loss": 3.4005, + "theoretical_loss": 4.460684613876367, + "tokens_seen": 165675008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047968906720160484, + "loss": 3.1465, + "theoretical_loss": 4.460457416257386, + "tokens_seen": 165740544 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479679037111334, + "loss": 2.865, + "theoretical_loss": 4.460230333600595, + "tokens_seen": 165806080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796690070210632, + "loss": 3.2027, + "theoretical_loss": 4.46000336580242, + "tokens_seen": 165871616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796589769307924, + "loss": 3.3585, + "theoretical_loss": 4.459776512759419, + "tokens_seen": 165937152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047964894684052157, + "loss": 3.2687, + "theoretical_loss": 4.459549774368284, + "tokens_seen": 166002688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796389167502508, + "loss": 3.4614, + "theoretical_loss": 4.45932315052584, + "tokens_seen": 166068224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047962888665997993, + "loss": 2.9803, + "theoretical_loss": 4.459096641129047, + "tokens_seen": 166133760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047961885656970917, + "loss": 3.0739, + "theoretical_loss": 4.458870246074998, + "tokens_seen": 166199296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047960882647943835, + "loss": 3.285, + "theoretical_loss": 4.458643965260917, + "tokens_seen": 166264832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047959879638916753, + "loss": 3.1638, + "theoretical_loss": 4.458417798584164, + "tokens_seen": 166330368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795887662988967, + "loss": 2.905, + "theoretical_loss": 4.45819174594223, + "tokens_seen": 166395904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795787362086259, + "loss": 3.2569, + "theoretical_loss": 4.457965807232739, + "tokens_seen": 166461440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047956870611835507, + "loss": 3.299, + "theoretical_loss": 4.457739982353445, + "tokens_seen": 166526976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795586760280843, + "loss": 2.9863, + "theoretical_loss": 4.457514271202237, + "tokens_seen": 166592512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047954864593781343, + "loss": 3.2391, + "theoretical_loss": 4.457288673677134, + "tokens_seen": 166658048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953861584754267, + "loss": 3.3358, + "theoretical_loss": 4.457063189676287, + "tokens_seen": 166723584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795285857572718, + "loss": 3.2532, + "theoretical_loss": 4.456837819097978, + "tokens_seen": 166789120 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047951855566700103, + "loss": 3.1618, + "theoretical_loss": 4.45661256184062, + "tokens_seen": 166854656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795085255767302, + "loss": 3.0743, + "theoretical_loss": 4.456387417802757, + "tokens_seen": 166920192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794984954864594, + "loss": 3.3956, + "theoretical_loss": 4.456162386883065, + "tokens_seen": 166985728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794884653961886, + "loss": 3.1311, + "theoretical_loss": 4.455937468980348, + "tokens_seen": 167051264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 103097, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.102741003036499, + "objective/train/theoretical_loss": 4.455712663993541, + "objective/train/tokens_used": 187576800, + "theoretical_loss": 4.455712663993541, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047947843530591776, + "loss": 3.2024, + "theoretical_loss": 4.455712663993541, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047946840521564694, + "loss": 3.4233, + "theoretical_loss": 4.455487971821709, + "tokens_seen": 167182336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047945837512537617, + "loss": 3.1938, + "theoretical_loss": 4.4552633923640474, + "tokens_seen": 167247872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794483450351053, + "loss": 3.1579, + "theoretical_loss": 4.455038925519882, + "tokens_seen": 167313408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943831494483453, + "loss": 3.2525, + "theoretical_loss": 4.454814571188665, + "tokens_seen": 167378944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794282848545637, + "loss": 3.0859, + "theoretical_loss": 4.454590329269979, + "tokens_seen": 167444480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794182547642929, + "loss": 3.4173, + "theoretical_loss": 4.454366199663537, + "tokens_seen": 167510016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794082246740221, + "loss": 3.1989, + "theoretical_loss": 4.4541421822691785, + "tokens_seen": 167575552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939819458375126, + "loss": 3.2261, + "theoretical_loss": 4.453918276986872, + "tokens_seen": 167641088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047938816449348044, + "loss": 3.3857, + "theoretical_loss": 4.453694483716717, + "tokens_seen": 167706624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793781344032097, + "loss": 3.2033, + "theoretical_loss": 4.453470802358938, + "tokens_seen": 167772160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793681043129388, + "loss": 3.3288, + "theoretical_loss": 4.453247232813885, + "tokens_seen": 167837696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047935807422266804, + "loss": 3.3873, + "theoretical_loss": 4.453023774982041, + "tokens_seen": 167903232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047934804413239716, + "loss": 3.027, + "theoretical_loss": 4.4528004287640135, + "tokens_seen": 167968768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793380140421264, + "loss": 2.93, + "theoretical_loss": 4.452577194060538, + "tokens_seen": 168034304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793279839518556, + "loss": 3.338, + "theoretical_loss": 4.452354070772476, + "tokens_seen": 168099840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047931795386158476, + "loss": 3.0985, + "theoretical_loss": 4.452131058800816, + "tokens_seen": 168165376 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047930792377131394, + "loss": 3.0258, + "theoretical_loss": 4.451908158046674, + "tokens_seen": 168230912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792978936810431, + "loss": 3.2144, + "theoretical_loss": 4.451685368411292, + "tokens_seen": 168296448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792878635907723, + "loss": 3.1754, + "theoretical_loss": 4.4514626897960365, + "tokens_seen": 168361984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047927783350050154, + "loss": 2.8836, + "theoretical_loss": 4.451240122102401, + "tokens_seen": 168427520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047926780341023067, + "loss": 3.2178, + "theoretical_loss": 4.451017665232007, + "tokens_seen": 168493056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792577733199599, + "loss": 2.9942, + "theoretical_loss": 4.4507953190865965, + "tokens_seen": 168558592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792477432296891, + "loss": 3.2664, + "theoretical_loss": 4.450573083568042, + "tokens_seen": 168624128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923771313941827, + "loss": 2.9989, + "theoretical_loss": 4.450350958578337, + "tokens_seen": 168689664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 104347, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.790850877761841, + "objective/train/theoretical_loss": 4.4501289440196015, + "objective/train/tokens_used": 189215200, + "theoretical_loss": 4.4501289440196015, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047922768304914745, + "loss": 3.2164, + "theoretical_loss": 4.4501289440196015, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047921765295887663, + "loss": 3.3929, + "theoretical_loss": 4.449907039794081, + "tokens_seen": 168820736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792076228686058, + "loss": 3.2061, + "theoretical_loss": 4.449685245804145, + "tokens_seen": 168886272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919759277833504, + "loss": 2.9032, + "theoretical_loss": 4.449463561952285, + "tokens_seen": 168951808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047918756268806417, + "loss": 3.1277, + "theoretical_loss": 4.44924198814112, + "tokens_seen": 169017344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791775325977934, + "loss": 3.2685, + "theoretical_loss": 4.44902052427339, + "tokens_seen": 169082880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047916750250752253, + "loss": 3.0655, + "theoretical_loss": 4.448799170251961, + "tokens_seen": 169148416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047915747241725177, + "loss": 3.3278, + "theoretical_loss": 4.448577925979821, + "tokens_seen": 169213952 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047914744232698095, + "loss": 3.2436, + "theoretical_loss": 4.44835679136008, + "tokens_seen": 169279488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913741223671013, + "loss": 3.2687, + "theoretical_loss": 4.448135766295974, + "tokens_seen": 169345024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791273821464393, + "loss": 3.1304, + "theoretical_loss": 4.447914850690859, + "tokens_seen": 169410560 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047911735205616855, + "loss": 3.199, + "theoretical_loss": 4.447694044448216, + "tokens_seen": 169476096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791073219658977, + "loss": 3.2522, + "theoretical_loss": 4.447473347471645, + "tokens_seen": 169541632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790972918756269, + "loss": 3.1969, + "theoretical_loss": 4.447252759664874, + "tokens_seen": 169607168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047908726178535604, + "loss": 3.0039, + "theoretical_loss": 4.4470322809317455, + "tokens_seen": 169672704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047907723169508527, + "loss": 3.5408, + "theoretical_loss": 4.44681191117623, + "tokens_seen": 169738240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047906720160481445, + "loss": 3.2186, + "theoretical_loss": 4.446591650302416, + "tokens_seen": 169803776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905717151454363, + "loss": 2.9781, + "theoretical_loss": 4.446371498214516, + "tokens_seen": 169869312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790471414242728, + "loss": 3.4193, + "theoretical_loss": 4.446151454816862, + "tokens_seen": 169934848 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479037111334002, + "loss": 3.0771, + "theoretical_loss": 4.445931520013906, + "tokens_seen": 170000384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790270812437312, + "loss": 3.2464, + "theoretical_loss": 4.445711693710223, + "tokens_seen": 170065920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790170511534604, + "loss": 3.2014, + "theoretical_loss": 4.4454919758105085, + "tokens_seen": 170131456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047900702106318954, + "loss": 3.3265, + "theoretical_loss": 4.445272366219575, + "tokens_seen": 170196992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789969909729188, + "loss": 3.073, + "theoretical_loss": 4.44505286484236, + "tokens_seen": 170262528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789869608826479, + "loss": 2.936, + "theoretical_loss": 4.4448334715839195, + "tokens_seen": 170328064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 105037, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8210434913635254, + "objective/train/theoretical_loss": 4.444614186349425, + "objective/train/tokens_used": 190853600, + "theoretical_loss": 4.444614186349425, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047897693079237714, + "loss": 2.9571, + "theoretical_loss": 4.444614186349425, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047896690070210637, + "loss": 2.9778, + "theoretical_loss": 4.444395009044175, + "tokens_seen": 170459136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789568706118355, + "loss": 3.2967, + "theoretical_loss": 4.444175939573581, + "tokens_seen": 170524672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047894684052156473, + "loss": 2.8109, + "theoretical_loss": 4.443956977843177, + "tokens_seen": 170590208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789368104312939, + "loss": 3.4383, + "theoretical_loss": 4.4437381237586155, + "tokens_seen": 170655744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789267803410231, + "loss": 3.1635, + "theoretical_loss": 4.443519377225666, + "tokens_seen": 170721280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789167502507523, + "loss": 3.1526, + "theoretical_loss": 4.443300738150221, + "tokens_seen": 170786816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047890672016048146, + "loss": 3.3384, + "theoretical_loss": 4.4430822064382856, + "tokens_seen": 170852352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047889669007021064, + "loss": 3.4135, + "theoretical_loss": 4.442863781995989, + "tokens_seen": 170917888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788866599799399, + "loss": 3.0138, + "theoretical_loss": 4.442645464729573, + "tokens_seen": 170983424 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478876629889669, + "loss": 3.2596, + "theoretical_loss": 4.4424272545454, + "tokens_seen": 171048960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047886659979939824, + "loss": 3.2555, + "theoretical_loss": 4.442209151349951, + "tokens_seen": 171114496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047885656970912736, + "loss": 3.3042, + "theoretical_loss": 4.4419911550498234, + "tokens_seen": 171180032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788465396188566, + "loss": 3.2061, + "theoretical_loss": 4.441773265551731, + "tokens_seen": 171245568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788365095285858, + "loss": 3.0708, + "theoretical_loss": 4.441555482762507, + "tokens_seen": 171311104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047882647943831496, + "loss": 3.2816, + "theoretical_loss": 4.441337806589097, + "tokens_seen": 171376640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047881644934804414, + "loss": 3.1222, + "theoretical_loss": 4.441120236938568, + "tokens_seen": 171442176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788064192577733, + "loss": 3.263, + "theoretical_loss": 4.440902773718101, + "tokens_seen": 171507712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787963891675025, + "loss": 2.9574, + "theoretical_loss": 4.440685416834994, + "tokens_seen": 171573248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047878635907723174, + "loss": 3.4099, + "theoretical_loss": 4.440468166196662, + "tokens_seen": 171638784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047877632898696087, + "loss": 3.2833, + "theoretical_loss": 4.440251021710634, + "tokens_seen": 171704320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787662988966901, + "loss": 3.2847, + "theoretical_loss": 4.440033983284556, + "tokens_seen": 171769856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787562688064193, + "loss": 3.3134, + "theoretical_loss": 4.43981705082619, + "tokens_seen": 171835392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047874623871614847, + "loss": 3.1022, + "theoretical_loss": 4.439600224243412, + "tokens_seen": 171900928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047873620862587765, + "loss": 3.1256, + "theoretical_loss": 4.439383503444214, + "tokens_seen": 171966464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 106163, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.448519468307495, + "objective/train/theoretical_loss": 4.439166888336702, + "objective/train/tokens_used": 192492000, + "theoretical_loss": 4.439166888336702, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047872617853560683, + "loss": 3.3453, + "theoretical_loss": 4.439166888336702, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478716148445336, + "loss": 3.0654, + "theoretical_loss": 4.438950378829098, + "tokens_seen": 172097536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047870611835506524, + "loss": 3.3586, + "theoretical_loss": 4.43873397482974, + "tokens_seen": 172163072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047869608826479437, + "loss": 3.1219, + "theoretical_loss": 4.438517676247075, + "tokens_seen": 172228608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786860581745236, + "loss": 3.0914, + "theoretical_loss": 4.4383014829896705, + "tokens_seen": 172294144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047867602808425273, + "loss": 3.1461, + "theoretical_loss": 4.4380853949662065, + "tokens_seen": 172359680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047866599799398197, + "loss": 3.0581, + "theoretical_loss": 4.4378694120854725, + "tokens_seen": 172425216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047865596790371115, + "loss": 3.0474, + "theoretical_loss": 4.437653534256377, + "tokens_seen": 172490752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047864593781344033, + "loss": 3.207, + "theoretical_loss": 4.437437761387939, + "tokens_seen": 172556288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786359077231695, + "loss": 3.1347, + "theoretical_loss": 4.437222093389293, + "tokens_seen": 172621824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047862587763289875, + "loss": 3.0996, + "theoretical_loss": 4.437006530169684, + "tokens_seen": 172687360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786158475426279, + "loss": 3.255, + "theoretical_loss": 4.436791071638472, + "tokens_seen": 172752896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786058174523571, + "loss": 3.3615, + "theoretical_loss": 4.43657571770513, + "tokens_seen": 172818432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047859578736208624, + "loss": 3.3575, + "theoretical_loss": 4.436360468279242, + "tokens_seen": 172883968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047858575727181547, + "loss": 3.1795, + "theoretical_loss": 4.4361453232705035, + "tokens_seen": 172949504 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047857572718154465, + "loss": 3.2246, + "theoretical_loss": 4.435930282588725, + "tokens_seen": 173015040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047856569709127383, + "loss": 3.1589, + "theoretical_loss": 4.435715346143828, + "tokens_seen": 173080576 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478555667001003, + "loss": 3.1079, + "theoretical_loss": 4.435500513845845, + "tokens_seen": 173146112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785456369107322, + "loss": 3.0886, + "theoretical_loss": 4.435285785604922, + "tokens_seen": 173211648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785356068204614, + "loss": 3.3142, + "theoretical_loss": 4.435071161331313, + "tokens_seen": 173277184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785255767301906, + "loss": 3.3968, + "theoretical_loss": 4.434856640935388, + "tokens_seen": 173342720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047851554663991974, + "loss": 3.1791, + "theoretical_loss": 4.434642224327625, + "tokens_seen": 173408256 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478505516549649, + "loss": 3.1941, + "theoretical_loss": 4.434427911418613, + "tokens_seen": 173473792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784954864593781, + "loss": 3.1568, + "theoretical_loss": 4.434213702119052, + "tokens_seen": 173539328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047848545636910734, + "loss": 3.3522, + "theoretical_loss": 4.433999596339755, + "tokens_seen": 173604864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 106755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8909780979156494, + "objective/train/theoretical_loss": 4.433785593991642, + "objective/train/tokens_used": 194130400, + "theoretical_loss": 4.433785593991642, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784754262788365, + "loss": 3.2649, + "theoretical_loss": 4.433785593991642, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784653961885657, + "loss": 3.2448, + "theoretical_loss": 4.433571694985744, + "tokens_seen": 173735936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784553660982949, + "loss": 3.2039, + "theoretical_loss": 4.433357899233204, + "tokens_seen": 173801472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784453360080241, + "loss": 3.0269, + "theoretical_loss": 4.433144206645274, + "tokens_seen": 173867008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047843530591775324, + "loss": 3.0917, + "theoretical_loss": 4.432930617133313, + "tokens_seen": 173932544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784252758274825, + "loss": 3.1783, + "theoretical_loss": 4.432717130608793, + "tokens_seen": 173998080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784152457372116, + "loss": 3.1232, + "theoretical_loss": 4.432503746983295, + "tokens_seen": 174063616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047840521564694084, + "loss": 3.2631, + "theoretical_loss": 4.432290466168507, + "tokens_seen": 174129152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047839518555667, + "loss": 3.2136, + "theoretical_loss": 4.432077288076228, + "tokens_seen": 174194688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783851554663992, + "loss": 3.2796, + "theoretical_loss": 4.431864212618365, + "tokens_seen": 174260224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783751253761284, + "loss": 3.4694, + "theoretical_loss": 4.431651239706934, + "tokens_seen": 174325760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047836509528585756, + "loss": 3.0788, + "theoretical_loss": 4.431438369254058, + "tokens_seen": 174391296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047835506519558675, + "loss": 3.0872, + "theoretical_loss": 4.43122560117197, + "tokens_seen": 174456832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478345035105316, + "loss": 2.9119, + "theoretical_loss": 4.431012935373012, + "tokens_seen": 174522368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783350050150451, + "loss": 3.2154, + "theoretical_loss": 4.430800371769631, + "tokens_seen": 174587904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047832497492477434, + "loss": 3.2817, + "theoretical_loss": 4.430587910274383, + "tokens_seen": 174653440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047831494483450347, + "loss": 3.1007, + "theoretical_loss": 4.430375550799933, + "tokens_seen": 174718976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783049147442327, + "loss": 3.3202, + "theoretical_loss": 4.430163293259053, + "tokens_seen": 174784512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782948846539619, + "loss": 3.2775, + "theoretical_loss": 4.42995113756462, + "tokens_seen": 174850048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047828485456369107, + "loss": 3.4485, + "theoretical_loss": 4.42973908362962, + "tokens_seen": 174915584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047827482447342025, + "loss": 3.1023, + "theoretical_loss": 4.429527131367147, + "tokens_seen": 174981120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782647943831495, + "loss": 3.0822, + "theoretical_loss": 4.429315280690398, + "tokens_seen": 175046656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782547642928786, + "loss": 3.2901, + "theoretical_loss": 4.4291035315126805, + "tokens_seen": 175112192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047824473420260785, + "loss": 3.2442, + "theoretical_loss": 4.428891883747406, + "tokens_seen": 175177728 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478234704112337, + "loss": 3.166, + "theoretical_loss": 4.428680337308094, + "tokens_seen": 175243264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 107724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6482720375061035, + "objective/train/theoretical_loss": 4.428468892108368, + "objective/train/tokens_used": 195768800, + "theoretical_loss": 4.428468892108368, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782246740220662, + "loss": 3.3804, + "theoretical_loss": 4.428468892108368, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047821464393179544, + "loss": 3.4307, + "theoretical_loss": 4.428257548061959, + "tokens_seen": 175374336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047820461384152457, + "loss": 3.2556, + "theoretical_loss": 4.428046305082704, + "tokens_seen": 175439872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781945837512538, + "loss": 3.3203, + "theoretical_loss": 4.427835163084543, + "tokens_seen": 175505408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047818455366098293, + "loss": 3.1723, + "theoretical_loss": 4.427624121981523, + "tokens_seen": 175570944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047817452357071217, + "loss": 3.1751, + "theoretical_loss": 4.427413181687799, + "tokens_seen": 175636480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047816449348044135, + "loss": 2.9857, + "theoretical_loss": 4.427202342117624, + "tokens_seen": 175702016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047815446339017053, + "loss": 3.1727, + "theoretical_loss": 4.426991603185365, + "tokens_seen": 175767552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781444332998997, + "loss": 3.121, + "theoretical_loss": 4.426780964805486, + "tokens_seen": 175833088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047813440320962895, + "loss": 3.3494, + "theoretical_loss": 4.4265704268925585, + "tokens_seen": 175898624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781243731193581, + "loss": 3.313, + "theoretical_loss": 4.426359989361259, + "tokens_seen": 175964160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781143430290873, + "loss": 3.3073, + "theoretical_loss": 4.426149652126369, + "tokens_seen": 176029696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047810431293881644, + "loss": 3.0604, + "theoretical_loss": 4.42593941510277, + "tokens_seen": 176095232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047809428284854567, + "loss": 2.7989, + "theoretical_loss": 4.42572927820545, + "tokens_seen": 176160768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047808425275827485, + "loss": 3.2073, + "theoretical_loss": 4.4255192413495035, + "tokens_seen": 176226304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047807422266800403, + "loss": 3.0824, + "theoretical_loss": 4.425309304450122, + "tokens_seen": 176291840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780641925777332, + "loss": 3.1597, + "theoretical_loss": 4.4250994674226085, + "tokens_seen": 176357376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780541624874624, + "loss": 3.1731, + "theoretical_loss": 4.424889730182361, + "tokens_seen": 176422912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780441323971916, + "loss": 3.3307, + "theoretical_loss": 4.424680092644886, + "tokens_seen": 176488448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780341023069208, + "loss": 3.0847, + "theoretical_loss": 4.42447055472579, + "tokens_seen": 176553984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047802407221664994, + "loss": 3.3959, + "theoretical_loss": 4.4242611163407854, + "tokens_seen": 176619520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780140421263792, + "loss": 3.2021, + "theoretical_loss": 4.424051777405684, + "tokens_seen": 176685056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780040120361083, + "loss": 3.0791, + "theoretical_loss": 4.423842537836402, + "tokens_seen": 176750592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799398194583754, + "loss": 3.112, + "theoretical_loss": 4.423633397548956, + "tokens_seen": 176816128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779839518555667, + "loss": 3.0609, + "theoretical_loss": 4.423424356459467, + "tokens_seen": 176881664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 108296, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4872632026672363, + "objective/train/theoretical_loss": 4.423215414484155, + "objective/train/tokens_used": 197407200, + "theoretical_loss": 4.423215414484155, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779739217652959, + "loss": 3.3284, + "theoretical_loss": 4.423215414484155, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779638916750251, + "loss": 2.839, + "theoretical_loss": 4.423006571539346, + "tokens_seen": 177012736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779538615847543, + "loss": 3.1684, + "theoretical_loss": 4.422797827541462, + "tokens_seen": 177078272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047794383149448344, + "loss": 3.1424, + "theoretical_loss": 4.422589182407032, + "tokens_seen": 177143808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779338014042127, + "loss": 3.2199, + "theoretical_loss": 4.422380636052681, + "tokens_seen": 177209344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779237713139418, + "loss": 2.9687, + "theoretical_loss": 4.422172188395141, + "tokens_seen": 177274880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047791374122367104, + "loss": 3.0297, + "theoretical_loss": 4.421963839351239, + "tokens_seen": 177340416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779037111334002, + "loss": 3.1726, + "theoretical_loss": 4.421755588837907, + "tokens_seen": 177405952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778936810431294, + "loss": 3.3507, + "theoretical_loss": 4.421547436772176, + "tokens_seen": 177471488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778836509528586, + "loss": 3.2595, + "theoretical_loss": 4.421339383071178, + "tokens_seen": 177537024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047787362086258776, + "loss": 3.3193, + "theoretical_loss": 4.421131427652145, + "tokens_seen": 177602560 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047786359077231695, + "loss": 3.2873, + "theoretical_loss": 4.420923570432407, + "tokens_seen": 177668096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778535606820462, + "loss": 3.1947, + "theoretical_loss": 4.420715811329398, + "tokens_seen": 177733632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778435305917753, + "loss": 3.2383, + "theoretical_loss": 4.420508150260651, + "tokens_seen": 177799168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783350050150454, + "loss": 3.1387, + "theoretical_loss": 4.420300587143795, + "tokens_seen": 177864704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047782347041123367, + "loss": 2.8977, + "theoretical_loss": 4.420093121896563, + "tokens_seen": 177930240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778134403209629, + "loss": 3.1988, + "theoretical_loss": 4.419885754436784, + "tokens_seen": 177995776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778034102306921, + "loss": 3.0306, + "theoretical_loss": 4.4196784846823896, + "tokens_seen": 178061312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047779338014042127, + "loss": 3.1273, + "theoretical_loss": 4.419471312551408, + "tokens_seen": 178126848 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047778335005015045, + "loss": 2.9744, + "theoretical_loss": 4.419264237961966, + "tokens_seen": 178192384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777733199598797, + "loss": 3.0912, + "theoretical_loss": 4.41905726083229, + "tokens_seen": 178257920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777632898696088, + "loss": 3.2138, + "theoretical_loss": 4.418850381080706, + "tokens_seen": 178323456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775325977933805, + "loss": 3.2271, + "theoretical_loss": 4.4186435986256365, + "tokens_seen": 178388992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777432296890672, + "loss": 3.0616, + "theoretical_loss": 4.418436913385604, + "tokens_seen": 178454528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777331995987964, + "loss": 3.2644, + "theoretical_loss": 4.418230325279228, + "tokens_seen": 178520064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 109542, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0579349994659424, + "objective/train/theoretical_loss": 4.418023834225226, + "objective/train/tokens_used": 199045600, + "theoretical_loss": 4.418023834225226, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777231695085256, + "loss": 3.1012, + "theoretical_loss": 4.418023834225226, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771313941825477, + "loss": 3.1625, + "theoretical_loss": 4.417817440142414, + "tokens_seen": 178651136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047770310932798395, + "loss": 3.0349, + "theoretical_loss": 4.417611142949705, + "tokens_seen": 178716672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047769307923771313, + "loss": 3.2227, + "theoretical_loss": 4.417404942566111, + "tokens_seen": 178782208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776830491474423, + "loss": 3.0844, + "theoretical_loss": 4.417198838910737, + "tokens_seen": 178847744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047767301905717155, + "loss": 3.3573, + "theoretical_loss": 4.4169928319027925, + "tokens_seen": 178913280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776629889669007, + "loss": 3.297, + "theoretical_loss": 4.416786921461577, + "tokens_seen": 178978816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776529588766299, + "loss": 3.2404, + "theoretical_loss": 4.41658110750649, + "tokens_seen": 179044352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047764292878635904, + "loss": 3.2353, + "theoretical_loss": 4.416375389957029, + "tokens_seen": 179109888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776328986960883, + "loss": 3.2372, + "theoretical_loss": 4.416169768732786, + "tokens_seen": 179175424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047762286860581746, + "loss": 3.1656, + "theoretical_loss": 4.415964243753448, + "tokens_seen": 179240960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761283851554664, + "loss": 2.8394, + "theoretical_loss": 4.415758814938803, + "tokens_seen": 179306496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776028084252758, + "loss": 3.3786, + "theoretical_loss": 4.415553482208733, + "tokens_seen": 179372032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047759277833500505, + "loss": 3.2626, + "theoretical_loss": 4.415348245483212, + "tokens_seen": 179437568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775827482447342, + "loss": 3.2584, + "theoretical_loss": 4.415143104682317, + "tokens_seen": 179503104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775727181544634, + "loss": 3.2606, + "theoretical_loss": 4.4149380597262144, + "tokens_seen": 179568640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047756268806419254, + "loss": 3.3959, + "theoretical_loss": 4.41473311053517, + "tokens_seen": 179634176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775526579739218, + "loss": 3.1822, + "theoretical_loss": 4.414528257029543, + "tokens_seen": 179699712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047754262788365096, + "loss": 3.3075, + "theoretical_loss": 4.414323499129791, + "tokens_seen": 179765248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753259779338014, + "loss": 3.1764, + "theoretical_loss": 4.414118836756462, + "tokens_seen": 179830784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775225677031093, + "loss": 3.2029, + "theoretical_loss": 4.4139142698302, + "tokens_seen": 179896320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775125376128385, + "loss": 3.0117, + "theoretical_loss": 4.413709798271748, + "tokens_seen": 179961856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775025075225677, + "loss": 3.0247, + "theoretical_loss": 4.41350542200194, + "tokens_seen": 180027392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774924774322969, + "loss": 3.2042, + "theoretical_loss": 4.413301140941703, + "tokens_seen": 180092928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047748244734202605, + "loss": 3.1478, + "theoretical_loss": 4.413096955012063, + "tokens_seen": 180158464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 110066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8472869396209717, + "objective/train/theoretical_loss": 4.412892864134137, + "objective/train/tokens_used": 200684000, + "theoretical_loss": 4.412892864134137, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774724172517553, + "loss": 3.0943, + "theoretical_loss": 4.412892864134137, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774623871614845, + "loss": 2.8881, + "theoretical_loss": 4.412688868229136, + "tokens_seen": 180289536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745235707121364, + "loss": 3.3796, + "theoretical_loss": 4.412484967218368, + "tokens_seen": 180355072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774423269809429, + "loss": 3.234, + "theoretical_loss": 4.4122811610232295, + "tokens_seen": 180420608 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477432296890672, + "loss": 3.1805, + "theoretical_loss": 4.412077449565216, + "tokens_seen": 180486144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047742226680040124, + "loss": 3.4462, + "theoretical_loss": 4.411873832765913, + "tokens_seen": 180551680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774122367101304, + "loss": 3.1975, + "theoretical_loss": 4.4116703105470005, + "tokens_seen": 180617216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774022066198596, + "loss": 3.1883, + "theoretical_loss": 4.411466882830252, + "tokens_seen": 180682752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773921765295888, + "loss": 3.1599, + "theoretical_loss": 4.411263549537534, + "tokens_seen": 180748288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047738214643931797, + "loss": 3.1214, + "theoretical_loss": 4.411060310590805, + "tokens_seen": 180813824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047737211634904715, + "loss": 3.0375, + "theoretical_loss": 4.410857165912116, + "tokens_seen": 180879360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773620862587764, + "loss": 3.0394, + "theoretical_loss": 4.410654115423613, + "tokens_seen": 180944896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773520561685055, + "loss": 3.0477, + "theoretical_loss": 4.410451159047533, + "tokens_seen": 181010432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047734202607823474, + "loss": 3.2956, + "theoretical_loss": 4.410248296706204, + "tokens_seen": 181075968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733199598796387, + "loss": 3.1011, + "theoretical_loss": 4.410045528322048, + "tokens_seen": 181141504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773219658976931, + "loss": 3.1455, + "theoretical_loss": 4.409842853817577, + "tokens_seen": 181207040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773119358074223, + "loss": 3.1878, + "theoretical_loss": 4.409640273115399, + "tokens_seen": 181272576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047730190571715147, + "loss": 2.8744, + "theoretical_loss": 4.409437786138211, + "tokens_seen": 181338112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729187562688065, + "loss": 3.1718, + "theoretical_loss": 4.4092353928087995, + "tokens_seen": 181403648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004772818455366099, + "loss": 3.1126, + "theoretical_loss": 4.409033093050046, + "tokens_seen": 181469184 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477271815446339, + "loss": 3.4133, + "theoretical_loss": 4.408830886784922, + "tokens_seen": 181534720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047726178535606825, + "loss": 3.2217, + "theoretical_loss": 4.40862877393649, + "tokens_seen": 181600256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772517552657974, + "loss": 3.1425, + "theoretical_loss": 4.408426754427904, + "tokens_seen": 181665792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772417251755266, + "loss": 3.009, + "theoretical_loss": 4.408224828182409, + "tokens_seen": 181731328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772316950852558, + "loss": 3.0163, + "theoretical_loss": 4.40802299512334, + "tokens_seen": 181796864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 111369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.439749002456665, + "objective/train/theoretical_loss": 4.407821255174123, + "objective/train/tokens_used": 202322400, + "theoretical_loss": 4.407821255174123, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047722166499498497, + "loss": 3.1176, + "theoretical_loss": 4.407821255174123, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047721163490471415, + "loss": 3.3191, + "theoretical_loss": 4.407619608258274, + "tokens_seen": 181927936 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047720160481444333, + "loss": 3.1351, + "theoretical_loss": 4.407418054299402, + "tokens_seen": 181993472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771915747241725, + "loss": 3.2347, + "theoretical_loss": 4.407216593221202, + "tokens_seen": 182059008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047718154463390175, + "loss": 3.182, + "theoretical_loss": 4.407015224947463, + "tokens_seen": 182124544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771715145436309, + "loss": 3.3172, + "theoretical_loss": 4.40681394940206, + "tokens_seen": 182190080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771614844533601, + "loss": 3.3428, + "theoretical_loss": 4.406612766508961, + "tokens_seen": 182255616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047715145436308924, + "loss": 3.1138, + "theoretical_loss": 4.4064116761922225, + "tokens_seen": 182321152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771414242728185, + "loss": 2.9224, + "theoretical_loss": 4.406210678375992, + "tokens_seen": 182386688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713139418254766, + "loss": 3.0207, + "theoretical_loss": 4.406009772984502, + "tokens_seen": 182452224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047712136409227684, + "loss": 2.9416, + "theoretical_loss": 4.40580895994208, + "tokens_seen": 182517760 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477111334002006, + "loss": 3.1591, + "theoretical_loss": 4.405608239173138, + "tokens_seen": 182583296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047710130391173525, + "loss": 3.142, + "theoretical_loss": 4.405407610602181, + "tokens_seen": 182648832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770912738214644, + "loss": 3.2257, + "theoretical_loss": 4.405207074153797, + "tokens_seen": 182714368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770812437311936, + "loss": 3.0585, + "theoretical_loss": 4.40500662975267, + "tokens_seen": 182779904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047707121364092274, + "loss": 3.0379, + "theoretical_loss": 4.4048062773235666, + "tokens_seen": 182845440 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477061183550652, + "loss": 3.018, + "theoretical_loss": 4.404606016791345, + "tokens_seen": 182910976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047705115346038116, + "loss": 3.1405, + "theoretical_loss": 4.404405848080951, + "tokens_seen": 182976512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047704112337011034, + "loss": 2.9844, + "theoretical_loss": 4.404205771117417, + "tokens_seen": 183042048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770310932798395, + "loss": 3.0418, + "theoretical_loss": 4.404005785825866, + "tokens_seen": 183107584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770210631895687, + "loss": 3.2314, + "theoretical_loss": 4.403805892131508, + "tokens_seen": 183173120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770110330992979, + "loss": 3.2265, + "theoretical_loss": 4.40360608995964, + "tokens_seen": 183238656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770010030090271, + "loss": 3.0284, + "theoretical_loss": 4.403406379235646, + "tokens_seen": 183304192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047699097291875625, + "loss": 3.0799, + "theoretical_loss": 4.403206759884999, + "tokens_seen": 183369728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769809428284855, + "loss": 3.0723, + "theoretical_loss": 4.403007231833261, + "tokens_seen": 183435264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 111925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.625623941421509, + "objective/train/theoretical_loss": 4.402807795006074, + "objective/train/tokens_used": 203960800, + "theoretical_loss": 4.402807795006074, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047697091273821466, + "loss": 3.2865, + "theoretical_loss": 4.402807795006074, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047696088264794384, + "loss": 2.8014, + "theoretical_loss": 4.402608449329178, + "tokens_seen": 183566336 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476950852557673, + "loss": 3.007, + "theoretical_loss": 4.40240919472839, + "tokens_seen": 183631872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769408224674022, + "loss": 3.0607, + "theoretical_loss": 4.40221003112962, + "tokens_seen": 183697408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769307923771314, + "loss": 3.393, + "theoretical_loss": 4.4020109584588605, + "tokens_seen": 183762944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769207622868606, + "loss": 3.3108, + "theoretical_loss": 4.401811976642195, + "tokens_seen": 183828480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047691073219658975, + "loss": 2.9578, + "theoretical_loss": 4.4016130856057885, + "tokens_seen": 183894016 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476900702106319, + "loss": 3.2152, + "theoretical_loss": 4.401414285275898, + "tokens_seen": 183959552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768906720160481, + "loss": 3.2252, + "theoretical_loss": 4.401215575578861, + "tokens_seen": 184025088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047688064192577735, + "loss": 3.0185, + "theoretical_loss": 4.4010169564411035, + "tokens_seen": 184090624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047687061183550653, + "loss": 3.0891, + "theoretical_loss": 4.4008184277891385, + "tokens_seen": 184156160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768605817452357, + "loss": 3.1172, + "theoretical_loss": 4.400619989549563, + "tokens_seen": 184221696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768505516549649, + "loss": 3.0661, + "theoretical_loss": 4.400421641649061, + "tokens_seen": 184287232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047684052156469407, + "loss": 2.9647, + "theoretical_loss": 4.400223384014399, + "tokens_seen": 184352768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047683049147442325, + "loss": 3.0501, + "theoretical_loss": 4.400025216572435, + "tokens_seen": 184418304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768204613841525, + "loss": 2.9332, + "theoretical_loss": 4.399827139250106, + "tokens_seen": 184483840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768104312938816, + "loss": 3.2431, + "theoretical_loss": 4.399629151974436, + "tokens_seen": 184549376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047680040120361085, + "loss": 2.9531, + "theoretical_loss": 4.399431254672535, + "tokens_seen": 184614912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047679037111334003, + "loss": 3.1576, + "theoretical_loss": 4.399233447271598, + "tokens_seen": 184680448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767803410230692, + "loss": 3.1937, + "theoretical_loss": 4.399035729698903, + "tokens_seen": 184745984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767703109327984, + "loss": 3.0762, + "theoretical_loss": 4.398838101881815, + "tokens_seen": 184811520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767602808425276, + "loss": 3.1412, + "theoretical_loss": 4.39864056374778, + "tokens_seen": 184877056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047675025075225676, + "loss": 3.2723, + "theoretical_loss": 4.398443115224333, + "tokens_seen": 184942592 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476740220661986, + "loss": 3.3477, + "theoretical_loss": 4.398245756239088, + "tokens_seen": 185008128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767301905717151, + "loss": 2.7878, + "theoretical_loss": 4.398048486719748, + "tokens_seen": 185073664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 113096, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7102208137512207, + "objective/train/theoretical_loss": 4.397851306594097, + "objective/train/tokens_used": 205599200, + "theoretical_loss": 4.397851306594097, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047672016048144435, + "loss": 2.8609, + "theoretical_loss": 4.397851306594097, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047671013039117353, + "loss": 3.0009, + "theoretical_loss": 4.397654215790003, + "tokens_seen": 185204736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767001003009027, + "loss": 3.029, + "theoretical_loss": 4.397457214235419, + "tokens_seen": 185270272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047669007021063195, + "loss": 3.292, + "theoretical_loss": 4.397260301858379, + "tokens_seen": 185335808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766800401203611, + "loss": 3.1928, + "theoretical_loss": 4.397063478587005, + "tokens_seen": 185401344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766700100300903, + "loss": 3.3045, + "theoretical_loss": 4.396866744349498, + "tokens_seen": 185466880 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047665997993981944, + "loss": 3.1277, + "theoretical_loss": 4.396670099074143, + "tokens_seen": 185532416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766499498495487, + "loss": 3.0829, + "theoretical_loss": 4.39647354268931, + "tokens_seen": 185597952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047663991975927786, + "loss": 3.2087, + "theoretical_loss": 4.39627707512345, + "tokens_seen": 185663488 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662988966900704, + "loss": 2.8462, + "theoretical_loss": 4.3960806963050985, + "tokens_seen": 185729024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766198595787362, + "loss": 3.0272, + "theoretical_loss": 4.395884406162873, + "tokens_seen": 185794560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047660982948846545, + "loss": 3.2038, + "theoretical_loss": 4.395688204625471, + "tokens_seen": 185860096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765997993981946, + "loss": 2.8081, + "theoretical_loss": 4.395492091621677, + "tokens_seen": 185925632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765897693079238, + "loss": 3.3035, + "theoretical_loss": 4.395296067080356, + "tokens_seen": 185991168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047657973921765294, + "loss": 3.1633, + "theoretical_loss": 4.3951001309304525, + "tokens_seen": 186056704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765697091273822, + "loss": 3.145, + "theoretical_loss": 4.394904283100997, + "tokens_seen": 186122240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047655967903711136, + "loss": 3.2733, + "theoretical_loss": 4.394708523521101, + "tokens_seen": 186187776 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654964894684054, + "loss": 3.1495, + "theoretical_loss": 4.394512852119956, + "tokens_seen": 186253312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765396188565697, + "loss": 3.1897, + "theoretical_loss": 4.3943172688268355, + "tokens_seen": 186318848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765295887662989, + "loss": 3.0105, + "theoretical_loss": 4.394121773571097, + "tokens_seen": 186384384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765195586760281, + "loss": 3.1493, + "theoretical_loss": 4.3939263662821775, + "tokens_seen": 186449920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765095285857573, + "loss": 2.9624, + "theoretical_loss": 4.393731046889595, + "tokens_seen": 186515456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047649949849548645, + "loss": 3.3138, + "theoretical_loss": 4.3935358153229505, + "tokens_seen": 186580992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764894684052157, + "loss": 3.1593, + "theoretical_loss": 4.3933406715119245, + "tokens_seen": 186646528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047647943831494486, + "loss": 3.0466, + "theoretical_loss": 4.393145615386279, + "tokens_seen": 186712064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 113525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.944962978363037, + "objective/train/theoretical_loss": 4.392950646875857, + "objective/train/tokens_used": 207237600, + "theoretical_loss": 4.392950646875857, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047646940822467404, + "loss": 3.0963, + "theoretical_loss": 4.392950646875857, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764593781344032, + "loss": 3.0612, + "theoretical_loss": 4.392755765910581, + "tokens_seen": 186843136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764493480441324, + "loss": 3.1482, + "theoretical_loss": 4.3925609724204575, + "tokens_seen": 186908672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764393179538616, + "loss": 3.2602, + "theoretical_loss": 4.392366266335569, + "tokens_seen": 186974208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764292878635908, + "loss": 3.1856, + "theoretical_loss": 4.392171647586081, + "tokens_seen": 187039744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047641925777331995, + "loss": 3.0132, + "theoretical_loss": 4.39197711610224, + "tokens_seen": 187105280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764092276830492, + "loss": 3.152, + "theoretical_loss": 4.391782671814369, + "tokens_seen": 187170816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763991975927783, + "loss": 3.2149, + "theoretical_loss": 4.391588314652875, + "tokens_seen": 187236352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047638916750250755, + "loss": 3.2574, + "theoretical_loss": 4.3913940445482424, + "tokens_seen": 187301888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047637913741223673, + "loss": 3.1899, + "theoretical_loss": 4.391199861431037, + "tokens_seen": 187367424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763691073219659, + "loss": 3.0975, + "theoretical_loss": 4.3910057652319034, + "tokens_seen": 187432960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763590772316951, + "loss": 3.2897, + "theoretical_loss": 4.390811755881566, + "tokens_seen": 187498496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047634904714142427, + "loss": 3.3427, + "theoretical_loss": 4.390617833310827, + "tokens_seen": 187564032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047633901705115345, + "loss": 3.181, + "theoretical_loss": 4.390423997450571, + "tokens_seen": 187629568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763289869608827, + "loss": 3.1201, + "theoretical_loss": 4.3902302482317594, + "tokens_seen": 187695104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763189568706118, + "loss": 3.12, + "theoretical_loss": 4.390036585585432, + "tokens_seen": 187760640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047630892678034105, + "loss": 2.9343, + "theoretical_loss": 4.3898430094427106, + "tokens_seen": 187826176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047629889669007023, + "loss": 3.0305, + "theoretical_loss": 4.389649519734793, + "tokens_seen": 187891712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762888665997994, + "loss": 3.0496, + "theoretical_loss": 4.3894561163929575, + "tokens_seen": 187957248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762788365095286, + "loss": 3.2563, + "theoretical_loss": 4.389262799348559, + "tokens_seen": 188022784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762688064192578, + "loss": 3.2352, + "theoretical_loss": 4.389069568533033, + "tokens_seen": 188088320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047625877632898696, + "loss": 3.1003, + "theoretical_loss": 4.388876423877893, + "tokens_seen": 188153856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762487462387162, + "loss": 3.1905, + "theoretical_loss": 4.388683365314728, + "tokens_seen": 188219392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762387161484453, + "loss": 3.3903, + "theoretical_loss": 4.388490392775209, + "tokens_seen": 188284928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047622868605817455, + "loss": 3.2064, + "theoretical_loss": 4.3882975061910825, + "tokens_seen": 188350464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 114652, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.638829469680786, + "objective/train/theoretical_loss": 4.3881047054941735, + "objective/train/tokens_used": 208876000, + "theoretical_loss": 4.3881047054941735, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762186559679037, + "loss": 3.0819, + "theoretical_loss": 4.3881047054941735, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762086258776329, + "loss": 2.914, + "theoretical_loss": 4.387911990616386, + "tokens_seen": 188481536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761985957873621, + "loss": 3.038, + "theoretical_loss": 4.387719361489699, + "tokens_seen": 188547072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761885656970913, + "loss": 2.9676, + "theoretical_loss": 4.3875268180461715, + "tokens_seen": 188612608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047617853560682046, + "loss": 3.2529, + "theoretical_loss": 4.387334360217939, + "tokens_seen": 188678144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047616850551654964, + "loss": 3.1968, + "theoretical_loss": 4.3871419879372135, + "tokens_seen": 188743680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761584754262788, + "loss": 3.3051, + "theoretical_loss": 4.3869497011362855, + "tokens_seen": 188809216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614844533600806, + "loss": 3.1021, + "theoretical_loss": 4.386757499747521, + "tokens_seen": 188874752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761384152457372, + "loss": 2.8122, + "theoretical_loss": 4.386565383703366, + "tokens_seen": 188940288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761283851554664, + "loss": 3.308, + "theoretical_loss": 4.38637335293634, + "tokens_seen": 189005824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761183550651956, + "loss": 3.2071, + "theoretical_loss": 4.3861814073790395, + "tokens_seen": 189071360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761083249749248, + "loss": 3.0571, + "theoretical_loss": 4.38598954696414, + "tokens_seen": 189136896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047609829488465396, + "loss": 3.2339, + "theoretical_loss": 4.385797771624393, + "tokens_seen": 189202432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047608826479438314, + "loss": 2.9145, + "theoretical_loss": 4.385606081292623, + "tokens_seen": 189267968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760782347041123, + "loss": 2.9952, + "theoretical_loss": 4.385414475901737, + "tokens_seen": 189333504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047606820461384156, + "loss": 3.4306, + "theoretical_loss": 4.385222955384711, + "tokens_seen": 189399040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760581745235707, + "loss": 3.2663, + "theoretical_loss": 4.385031519674601, + "tokens_seen": 189464576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760481444332999, + "loss": 3.267, + "theoretical_loss": 4.38484016870454, + "tokens_seen": 189530112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047603811434302905, + "loss": 3.1402, + "theoretical_loss": 4.384648902407734, + "tokens_seen": 189595648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760280842527583, + "loss": 3.3339, + "theoretical_loss": 4.384457720717466, + "tokens_seen": 189661184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047601805416248746, + "loss": 3.3079, + "theoretical_loss": 4.384266623567096, + "tokens_seen": 189726720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047600802407221665, + "loss": 3.0988, + "theoretical_loss": 4.384075610890057, + "tokens_seen": 189792256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759979939819458, + "loss": 3.1992, + "theoretical_loss": 4.3838846826198585, + "tokens_seen": 189857792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047598796389167506, + "loss": 3.1131, + "theoretical_loss": 4.383693838690085, + "tokens_seen": 189923328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759779338014042, + "loss": 3.2468, + "theoretical_loss": 4.383503079034397, + "tokens_seen": 189988864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 115651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9877991676330566, + "objective/train/theoretical_loss": 4.383312403586528, + "objective/train/tokens_used": 210514400, + "theoretical_loss": 4.383312403586528, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759679037111334, + "loss": 3.0685, + "theoretical_loss": 4.383312403586528, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759578736208626, + "loss": 2.9333, + "theoretical_loss": 4.38312181228029, + "tokens_seen": 190119936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759478435305918, + "loss": 3.366, + "theoretical_loss": 4.382931305049566, + "tokens_seen": 190185472 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475937813440321, + "loss": 3.087, + "theoretical_loss": 4.382740881828316, + "tokens_seen": 190251008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047592778335005015, + "loss": 3.0998, + "theoretical_loss": 4.3825505425505735, + "tokens_seen": 190316544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759177532597794, + "loss": 3.172, + "theoretical_loss": 4.382360287150449, + "tokens_seen": 190382080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759077231695085, + "loss": 3.2168, + "theoretical_loss": 4.382170115562122, + "tokens_seen": 190447616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047589769307923775, + "loss": 3.2619, + "theoretical_loss": 4.381980027719852, + "tokens_seen": 190513152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047588766298896693, + "loss": 3.1796, + "theoretical_loss": 4.3817900235579685, + "tokens_seen": 190578688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758776328986961, + "loss": 3.0825, + "theoretical_loss": 4.381600103010879, + "tokens_seen": 190644224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758676028084253, + "loss": 2.9786, + "theoretical_loss": 4.381410266013061, + "tokens_seen": 190709760 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047585757271815447, + "loss": 3.2466, + "theoretical_loss": 4.381220512499068, + "tokens_seen": 190775296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047584754262788365, + "loss": 3.0826, + "theoretical_loss": 4.381030842403527, + "tokens_seen": 190840832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758375125376129, + "loss": 3.2852, + "theoretical_loss": 4.38084125566114, + "tokens_seen": 190906368 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475827482447342, + "loss": 3.3628, + "theoretical_loss": 4.380651752206679, + "tokens_seen": 190971904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047581745235707125, + "loss": 2.9071, + "theoretical_loss": 4.380462331974991, + "tokens_seen": 191037440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047580742226680043, + "loss": 2.9733, + "theoretical_loss": 4.380272994900998, + "tokens_seen": 191102976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757973921765296, + "loss": 3.2946, + "theoretical_loss": 4.380083740919693, + "tokens_seen": 191168512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757873620862588, + "loss": 3.0393, + "theoretical_loss": 4.379894569966144, + "tokens_seen": 191234048 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475777331995988, + "loss": 3.2391, + "theoretical_loss": 4.37970548197549, + "tokens_seen": 191299584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047576730190571716, + "loss": 3.0312, + "theoretical_loss": 4.3795164768829435, + "tokens_seen": 191365120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757572718154464, + "loss": 3.0777, + "theoretical_loss": 4.37932755462379, + "tokens_seen": 191430656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757472417251755, + "loss": 3.3053, + "theoretical_loss": 4.379138715133389, + "tokens_seen": 191496192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047573721163490475, + "loss": 3.0129, + "theoretical_loss": 4.378949958347171, + "tokens_seen": 191561728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757271815446339, + "loss": 3.1993, + "theoretical_loss": 4.378761284200638, + "tokens_seen": 191627264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 116251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.686540126800537, + "objective/train/theoretical_loss": 4.3785726926293655, + "objective/train/tokens_used": 212152800, + "theoretical_loss": 4.3785726926293655, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757171514543631, + "loss": 2.8721, + "theoretical_loss": 4.3785726926293655, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757071213640923, + "loss": 3.144, + "theoretical_loss": 4.378384183569002, + "tokens_seen": 191758336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756970912738215, + "loss": 3.2052, + "theoretical_loss": 4.378195756955267, + "tokens_seen": 191823872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047568706118355066, + "loss": 3.192, + "theoretical_loss": 4.378007412723953, + "tokens_seen": 191889408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047567703109327984, + "loss": 2.9825, + "theoretical_loss": 4.377819150810923, + "tokens_seen": 191954944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475667001003009, + "loss": 2.9849, + "theoretical_loss": 4.377630971152113, + "tokens_seen": 192020480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047565697091273826, + "loss": 3.2592, + "theoretical_loss": 4.377442873683529, + "tokens_seen": 192086016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756469408224674, + "loss": 3.0667, + "theoretical_loss": 4.377254858341252, + "tokens_seen": 192151552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756369107321966, + "loss": 3.194, + "theoretical_loss": 4.377066925061431, + "tokens_seen": 192217088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756268806419258, + "loss": 3.2189, + "theoretical_loss": 4.376879073780288, + "tokens_seen": 192282624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475616850551655, + "loss": 3.0755, + "theoretical_loss": 4.376691304434117, + "tokens_seen": 192348160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047560682046138416, + "loss": 3.1954, + "theoretical_loss": 4.376503616959281, + "tokens_seen": 192413696 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047559679037111334, + "loss": 3.233, + "theoretical_loss": 4.376316011292214, + "tokens_seen": 192479232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755867602808425, + "loss": 3.0876, + "theoretical_loss": 4.376128487369426, + "tokens_seen": 192544768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047557673019057176, + "loss": 2.9403, + "theoretical_loss": 4.375941045127492, + "tokens_seen": 192610304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755667001003009, + "loss": 3.3601, + "theoretical_loss": 4.375753684503059, + "tokens_seen": 192675840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755566700100301, + "loss": 2.9828, + "theoretical_loss": 4.3755664054328465, + "tokens_seen": 192741376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047554663991975925, + "loss": 2.8375, + "theoretical_loss": 4.375379207853644, + "tokens_seen": 192806912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755366098294885, + "loss": 3.323, + "theoretical_loss": 4.375192091702311, + "tokens_seen": 192872448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047552657973921766, + "loss": 3.0783, + "theoretical_loss": 4.375005056915777, + "tokens_seen": 192937984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047551654964894685, + "loss": 2.974, + "theoretical_loss": 4.374818103431042, + "tokens_seen": 193003520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047550651955867603, + "loss": 3.0758, + "theoretical_loss": 4.374631231185177, + "tokens_seen": 193069056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047549648946840526, + "loss": 3.0464, + "theoretical_loss": 4.374444440115322, + "tokens_seen": 193134592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754864593781344, + "loss": 3.1286, + "theoretical_loss": 4.374257730158687, + "tokens_seen": 193200128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754764292878636, + "loss": 3.1677, + "theoretical_loss": 4.374071101252554, + "tokens_seen": 193265664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 117431, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7149760723114014, + "objective/train/theoretical_loss": 4.373884553334271, + "objective/train/tokens_used": 213791200, + "theoretical_loss": 4.373884553334271, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047546639919759275, + "loss": 3.1442, + "theoretical_loss": 4.373884553334271, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475456369107322, + "loss": 3.1432, + "theoretical_loss": 4.373698086341258, + "tokens_seen": 193396736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047544633901705117, + "loss": 3.2882, + "theoretical_loss": 4.373511700211004, + "tokens_seen": 193462272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047543630892678035, + "loss": 3.1277, + "theoretical_loss": 4.373325394881069, + "tokens_seen": 193527808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047542627883650953, + "loss": 2.8819, + "theoretical_loss": 4.373139170289079, + "tokens_seen": 193593344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754162487462387, + "loss": 3.1914, + "theoretical_loss": 4.3729530263727305, + "tokens_seen": 193658880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754062186559679, + "loss": 3.0863, + "theoretical_loss": 4.372766963069793, + "tokens_seen": 193724416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047539618856569713, + "loss": 3.1611, + "theoretical_loss": 4.3725809803181, + "tokens_seen": 193789952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047538615847542625, + "loss": 3.1407, + "theoretical_loss": 4.372395078055554, + "tokens_seen": 193855488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753761283851555, + "loss": 3.2841, + "theoretical_loss": 4.372209256220131, + "tokens_seen": 193921024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753660982948846, + "loss": 3.2507, + "theoretical_loss": 4.3720235147498725, + "tokens_seen": 193986560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047535606820461385, + "loss": 3.2756, + "theoretical_loss": 4.371837853582887, + "tokens_seen": 194052096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047534603811434303, + "loss": 2.9721, + "theoretical_loss": 4.3716522726573555, + "tokens_seen": 194117632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753360080240722, + "loss": 3.1121, + "theoretical_loss": 4.371466771911524, + "tokens_seen": 194183168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753259779338014, + "loss": 3.07, + "theoretical_loss": 4.371281351283709, + "tokens_seen": 194248704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047531594784353063, + "loss": 3.0433, + "theoretical_loss": 4.371096010712295, + "tokens_seen": 194314240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047530591775325976, + "loss": 3.1014, + "theoretical_loss": 4.370910750135733, + "tokens_seen": 194379776 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475295887662989, + "loss": 3.3776, + "theoretical_loss": 4.370725569492545, + "tokens_seen": 194445312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752858575727181, + "loss": 3.2206, + "theoretical_loss": 4.370540468721318, + "tokens_seen": 194510848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047527582748244736, + "loss": 3.1334, + "theoretical_loss": 4.370355447760708, + "tokens_seen": 194576384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047526579739217654, + "loss": 3.1063, + "theoretical_loss": 4.370170506549439, + "tokens_seen": 194641920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752557673019057, + "loss": 3.133, + "theoretical_loss": 4.369985645026302, + "tokens_seen": 194707456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752457372116349, + "loss": 3.1316, + "theoretical_loss": 4.369800863130157, + "tokens_seen": 194772992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752357071213641, + "loss": 3.0326, + "theoretical_loss": 4.36961616079993, + "tokens_seen": 194838528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047522567703109326, + "loss": 3.2395, + "theoretical_loss": 4.3694315379746165, + "tokens_seen": 194904064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 118646, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6308515071868896, + "objective/train/theoretical_loss": 4.369246994593274, + "objective/train/tokens_used": 215429600, + "theoretical_loss": 4.369246994593274, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752156469408225, + "loss": 3.2026, + "theoretical_loss": 4.369246994593274, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752056168505517, + "loss": 3.152, + "theoretical_loss": 4.369062530595034, + "tokens_seen": 195035136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047519558676028086, + "loss": 2.9739, + "theoretical_loss": 4.368878145919091, + "tokens_seen": 195100672 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047518555667001004, + "loss": 3.0244, + "theoretical_loss": 4.368693840504706, + "tokens_seen": 195166208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751755265797392, + "loss": 2.816, + "theoretical_loss": 4.368509614291209, + "tokens_seen": 195231744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047516549648946846, + "loss": 3.0789, + "theoretical_loss": 4.368325467217998, + "tokens_seen": 195297280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751554663991976, + "loss": 3.1642, + "theoretical_loss": 4.368141399224534, + "tokens_seen": 195362816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751454363089268, + "loss": 2.9522, + "theoretical_loss": 4.367957410250345, + "tokens_seen": 195428352 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475135406218656, + "loss": 3.0048, + "theoretical_loss": 4.367773500235029, + "tokens_seen": 195493888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751253761283852, + "loss": 3.172, + "theoretical_loss": 4.367589669118248, + "tokens_seen": 195559424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047511534603811436, + "loss": 3.02, + "theoretical_loss": 4.367405916839729, + "tokens_seen": 195624960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047510531594784354, + "loss": 3.0327, + "theoretical_loss": 4.367222243339268, + "tokens_seen": 195690496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750952858575727, + "loss": 3.3324, + "theoretical_loss": 4.367038648556725, + "tokens_seen": 195756032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047508525576730196, + "loss": 3.2043, + "theoretical_loss": 4.366855132432028, + "tokens_seen": 195821568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750752256770311, + "loss": 3.2153, + "theoretical_loss": 4.366671694905169, + "tokens_seen": 195887104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750651955867603, + "loss": 3.2576, + "theoretical_loss": 4.366488335916207, + "tokens_seen": 195952640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047505516549648945, + "loss": 2.9306, + "theoretical_loss": 4.366305055405267, + "tokens_seen": 196018176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750451354062187, + "loss": 3.163, + "theoretical_loss": 4.366121853312537, + "tokens_seen": 196083712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047503510531594787, + "loss": 3.2586, + "theoretical_loss": 4.365938729578274, + "tokens_seen": 196149248 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047502507522567705, + "loss": 3.313, + "theoretical_loss": 4.365755684142799, + "tokens_seen": 196214784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047501504513540623, + "loss": 3.2721, + "theoretical_loss": 4.365572716946499, + "tokens_seen": 196280320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047500501504513546, + "loss": 3.3105, + "theoretical_loss": 4.365389827929825, + "tokens_seen": 196345856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749949849548646, + "loss": 3.2141, + "theoretical_loss": 4.365207017033293, + "tokens_seen": 196411392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749849548645938, + "loss": 3.2766, + "theoretical_loss": 4.365024284197486, + "tokens_seen": 196476928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047497492477432295, + "loss": 2.9995, + "theoretical_loss": 4.364841629363051, + "tokens_seen": 196542464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 119101, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2888681888580322, + "objective/train/theoretical_loss": 4.364659052470699, + "objective/train/tokens_used": 217068000, + "theoretical_loss": 4.364659052470699, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749648946840522, + "loss": 3.1609, + "theoretical_loss": 4.364659052470699, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047495486459378137, + "loss": 3.1296, + "theoretical_loss": 4.364476553461208, + "tokens_seen": 196673536 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494483450351055, + "loss": 3.062, + "theoretical_loss": 4.364294132275419, + "tokens_seen": 196739072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047493480441323973, + "loss": 3.1813, + "theoretical_loss": 4.364111788854237, + "tokens_seen": 196804608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749247743229689, + "loss": 3.0742, + "theoretical_loss": 4.3639295231386335, + "tokens_seen": 196870144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749147442326981, + "loss": 3.1893, + "theoretical_loss": 4.363747335069642, + "tokens_seen": 196935680 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047490471414242733, + "loss": 3.2753, + "theoretical_loss": 4.363565224588364, + "tokens_seen": 197001216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047489468405215646, + "loss": 3.1104, + "theoretical_loss": 4.363383191635961, + "tokens_seen": 197066752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748846539618857, + "loss": 3.0305, + "theoretical_loss": 4.363201236153663, + "tokens_seen": 197132288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748746238716148, + "loss": 3.0246, + "theoretical_loss": 4.363019358082759, + "tokens_seen": 197197824 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047486459378134405, + "loss": 3.1562, + "theoretical_loss": 4.362837557364608, + "tokens_seen": 197263360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047485456369107323, + "loss": 3.1837, + "theoretical_loss": 4.362655833940627, + "tokens_seen": 197328896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748445336008024, + "loss": 3.2525, + "theoretical_loss": 4.3624741877523, + "tokens_seen": 197394432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748345035105316, + "loss": 3.1681, + "theoretical_loss": 4.362292618741174, + "tokens_seen": 197459968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047482447342026083, + "loss": 3.4425, + "theoretical_loss": 4.362111126848862, + "tokens_seen": 197525504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047481444332998996, + "loss": 3.3758, + "theoretical_loss": 4.3619297120170355, + "tokens_seen": 197591040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748044132397192, + "loss": 3.1236, + "theoretical_loss": 4.361748374187433, + "tokens_seen": 197656576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747943831494483, + "loss": 2.9103, + "theoretical_loss": 4.361567113301858, + "tokens_seen": 197722112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478435305917756, + "loss": 3.3328, + "theoretical_loss": 4.3613859293021715, + "tokens_seen": 197787648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047477432296890674, + "loss": 3.2619, + "theoretical_loss": 4.361204822130304, + "tokens_seen": 197853184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747642928786359, + "loss": 2.9688, + "theoretical_loss": 4.361023791728243, + "tokens_seen": 197918720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747542627883651, + "loss": 2.8798, + "theoretical_loss": 4.360842838038045, + "tokens_seen": 197984256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747442326980943, + "loss": 3.164, + "theoretical_loss": 4.360661961001824, + "tokens_seen": 198049792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047473420260782346, + "loss": 3.2726, + "theoretical_loss": 4.360481160561761, + "tokens_seen": 198115328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747241725175527, + "loss": 3.2962, + "theoretical_loss": 4.360300436660099, + "tokens_seen": 198180864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 119264, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4219841957092285, + "objective/train/theoretical_loss": 4.36011978923914, + "objective/train/tokens_used": 218706400, + "theoretical_loss": 4.36011978923914, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747141424272818, + "loss": 3.2649, + "theoretical_loss": 4.36011978923914, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470411233701106, + "loss": 3.1434, + "theoretical_loss": 4.359939218241253, + "tokens_seen": 198311936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746940822467402, + "loss": 3.3527, + "theoretical_loss": 4.359758723608866, + "tokens_seen": 198377472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746840521564694, + "loss": 3.2825, + "theoretical_loss": 4.359578305284474, + "tokens_seen": 198443008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746740220661986, + "loss": 3.3552, + "theoretical_loss": 4.359397963210629, + "tokens_seen": 198508544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746639919759278, + "loss": 3.2728, + "theoretical_loss": 4.3592176973299495, + "tokens_seen": 198574080 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047465396188565696, + "loss": 2.9474, + "theoretical_loss": 4.359037507585111, + "tokens_seen": 198639616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746439317953862, + "loss": 2.9234, + "theoretical_loss": 4.358857393918857, + "tokens_seen": 198705152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746339017051153, + "loss": 2.9143, + "theoretical_loss": 4.358677356273988, + "tokens_seen": 198770688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047462387161484456, + "loss": 3.1565, + "theoretical_loss": 4.358497394593371, + "tokens_seen": 198836224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746138415245737, + "loss": 3.118, + "theoretical_loss": 4.35831750881993, + "tokens_seen": 198901760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746038114343029, + "loss": 3.2261, + "theoretical_loss": 4.358137698896653, + "tokens_seen": 198967296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745937813440321, + "loss": 3.1764, + "theoretical_loss": 4.35795796476659, + "tokens_seen": 199032832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745837512537613, + "loss": 3.3871, + "theoretical_loss": 4.357778306372852, + "tokens_seen": 199098368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047457372116349047, + "loss": 3.1078, + "theoretical_loss": 4.357598723658612, + "tokens_seen": 199163904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047456369107321965, + "loss": 3.0791, + "theoretical_loss": 4.357419216567102, + "tokens_seen": 199229440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047455366098294883, + "loss": 3.2517, + "theoretical_loss": 4.357239785041618, + "tokens_seen": 199294976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454363089267807, + "loss": 3.176, + "theoretical_loss": 4.357060429025516, + "tokens_seen": 199360512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745336008024072, + "loss": 3.1573, + "theoretical_loss": 4.356881148462213, + "tokens_seen": 199426048 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047452357071213643, + "loss": 2.9391, + "theoretical_loss": 4.356701943295186, + "tokens_seen": 199491584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047451354062186555, + "loss": 3.0178, + "theoretical_loss": 4.356522813467976, + "tokens_seen": 199557120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745035105315948, + "loss": 3.008, + "theoretical_loss": 4.356343758924181, + "tokens_seen": 199622656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047449348044132397, + "loss": 3.088, + "theoretical_loss": 4.356164779607463, + "tokens_seen": 199688192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047448345035105315, + "loss": 3.0246, + "theoretical_loss": 4.355985875461542, + "tokens_seen": 199753728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744734202607824, + "loss": 3.2933, + "theoretical_loss": 4.3558070464302, + "tokens_seen": 199819264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 120457, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7158637046813965, + "objective/train/theoretical_loss": 4.355628292457278, + "objective/train/tokens_used": 220344800, + "theoretical_loss": 4.355628292457278, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047446339017051157, + "loss": 3.2748, + "theoretical_loss": 4.355628292457278, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047445336008024075, + "loss": 3.3399, + "theoretical_loss": 4.355449613486681, + "tokens_seen": 199950336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444332998996993, + "loss": 2.9307, + "theoretical_loss": 4.355271009462371, + "tokens_seen": 200015872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744332998996991, + "loss": 2.8099, + "theoretical_loss": 4.355092480328369, + "tokens_seen": 200081408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744232698094283, + "loss": 3.2346, + "theoretical_loss": 4.35491402602876, + "tokens_seen": 200146944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047441323971915753, + "loss": 3.1045, + "theoretical_loss": 4.354735646507687, + "tokens_seen": 200212480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047440320962888666, + "loss": 3.1639, + "theoretical_loss": 4.354557341709354, + "tokens_seen": 200278016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743931795386159, + "loss": 3.1702, + "theoretical_loss": 4.354379111578022, + "tokens_seen": 200343552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474383149448345, + "loss": 3.1362, + "theoretical_loss": 4.3542009560580155, + "tokens_seen": 200409088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047437311935807425, + "loss": 3.1808, + "theoretical_loss": 4.354022875093716, + "tokens_seen": 200474624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436308926780343, + "loss": 3.2076, + "theoretical_loss": 4.353844868629566, + "tokens_seen": 200540160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743530591775326, + "loss": 3.1964, + "theoretical_loss": 4.353666936610068, + "tokens_seen": 200605696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743430290872618, + "loss": 3.0344, + "theoretical_loss": 4.3534890789797815, + "tokens_seen": 200671232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047433299899699103, + "loss": 3.0296, + "theoretical_loss": 4.353311295683328, + "tokens_seen": 200736768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432296890672016, + "loss": 3.0211, + "theoretical_loss": 4.353133586665388, + "tokens_seen": 200802304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743129388164494, + "loss": 3.2548, + "theoretical_loss": 4.352955951870699, + "tokens_seen": 200867840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743029087261785, + "loss": 3.09, + "theoretical_loss": 4.352778391244061, + "tokens_seen": 200933376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047429287863590776, + "loss": 2.9689, + "theoretical_loss": 4.352600904730329, + "tokens_seen": 200998912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047428284854563694, + "loss": 3.4101, + "theoretical_loss": 4.35242349227442, + "tokens_seen": 201064448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742728184553661, + "loss": 3.288, + "theoretical_loss": 4.35224615382131, + "tokens_seen": 201129984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742627883650953, + "loss": 3.1146, + "theoretical_loss": 4.352068889316032, + "tokens_seen": 201195520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742527582748245, + "loss": 3.0871, + "theoretical_loss": 4.351891698703679, + "tokens_seen": 201261056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047424272818455366, + "loss": 3.2115, + "theoretical_loss": 4.351714581929402, + "tokens_seen": 201326592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742326980942829, + "loss": 2.8483, + "theoretical_loss": 4.351537538938409, + "tokens_seen": 201392128 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474222668004012, + "loss": 3.0209, + "theoretical_loss": 4.351360569675971, + "tokens_seen": 201457664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 121711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.200042724609375, + "objective/train/theoretical_loss": 4.351183674087412, + "objective/train/tokens_used": 221983200, + "theoretical_loss": 4.351183674087412, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047421263791374126, + "loss": 3.0898, + "theoretical_loss": 4.351183674087412, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742026078234704, + "loss": 3.0786, + "theoretical_loss": 4.35100685211812, + "tokens_seen": 201588736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741925777331996, + "loss": 3.155, + "theoretical_loss": 4.350830103713535, + "tokens_seen": 201654272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741825476429288, + "loss": 3.1987, + "theoretical_loss": 4.350653428819159, + "tokens_seen": 201719808 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474172517552658, + "loss": 3.0476, + "theoretical_loss": 4.350476827380553, + "tokens_seen": 201785344 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416248746238716, + "loss": 3.157, + "theoretical_loss": 4.350300299343332, + "tokens_seen": 201850880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741524573721164, + "loss": 3.0758, + "theoretical_loss": 4.350123844653173, + "tokens_seen": 201916416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741424272818455, + "loss": 3.2307, + "theoretical_loss": 4.349947463255808, + "tokens_seen": 201981952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047413239719157476, + "loss": 3.1247, + "theoretical_loss": 4.349771155097029, + "tokens_seen": 202047488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741223671013039, + "loss": 3.1268, + "theoretical_loss": 4.349594920122682, + "tokens_seen": 202113024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741123370110331, + "loss": 3.1097, + "theoretical_loss": 4.349418758278675, + "tokens_seen": 202178560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741023069207623, + "loss": 3.139, + "theoretical_loss": 4.349242669510971, + "tokens_seen": 202244096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740922768304915, + "loss": 3.1241, + "theoretical_loss": 4.3490666537655915, + "tokens_seen": 202309632 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047408224674022067, + "loss": 3.1542, + "theoretical_loss": 4.348890710988615, + "tokens_seen": 202375168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047407221664994985, + "loss": 3.2895, + "theoretical_loss": 4.348714841126174, + "tokens_seen": 202440704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047406218655967903, + "loss": 3.1676, + "theoretical_loss": 4.348539044124465, + "tokens_seen": 202506240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047405215646940827, + "loss": 2.9643, + "theoretical_loss": 4.348363319929735, + "tokens_seen": 202571776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740421263791374, + "loss": 3.0266, + "theoretical_loss": 4.348187668488293, + "tokens_seen": 202637312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047403209628886663, + "loss": 3.0482, + "theoretical_loss": 4.3480120897465016, + "tokens_seen": 202702848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047402206619859575, + "loss": 3.044, + "theoretical_loss": 4.347836583650782, + "tokens_seen": 202768384 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474012036108325, + "loss": 3.1396, + "theoretical_loss": 4.347661150147611, + "tokens_seen": 202833920 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047400200601805417, + "loss": 3.2979, + "theoretical_loss": 4.347485789183523, + "tokens_seen": 202899456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047399197592778335, + "loss": 3.1927, + "theoretical_loss": 4.34731050070511, + "tokens_seen": 202964992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047398194583751253, + "loss": 3.0788, + "theoretical_loss": 4.347135284659017, + "tokens_seen": 203030528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047397191574724177, + "loss": 3.0052, + "theoretical_loss": 4.346960140991949, + "tokens_seen": 203096064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 122225, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3157236576080322, + "objective/train/theoretical_loss": 4.346785069650667, + "objective/train/tokens_used": 223621600, + "theoretical_loss": 4.346785069650667, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739618856569709, + "loss": 2.9543, + "theoretical_loss": 4.346785069650667, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047395185556670013, + "loss": 3.0566, + "theoretical_loss": 4.346610070581987, + "tokens_seen": 203227136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047394182547642926, + "loss": 2.9878, + "theoretical_loss": 4.34643514373278, + "tokens_seen": 203292672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739317953861585, + "loss": 3.0709, + "theoretical_loss": 4.346260289049978, + "tokens_seen": 203358208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739217652958877, + "loss": 3.1681, + "theoretical_loss": 4.346085506480564, + "tokens_seen": 203423744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047391173520561686, + "loss": 3.2437, + "theoretical_loss": 4.34591079597158, + "tokens_seen": 203489280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047390170511534604, + "loss": 3.3256, + "theoretical_loss": 4.345736157470122, + "tokens_seen": 203554816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738916750250752, + "loss": 3.1524, + "theoretical_loss": 4.345561590923343, + "tokens_seen": 203620352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738816449348044, + "loss": 3.3006, + "theoretical_loss": 4.345387096278453, + "tokens_seen": 203685888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047387161484453363, + "loss": 3.088, + "theoretical_loss": 4.345212673482715, + "tokens_seen": 203751424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047386158475426276, + "loss": 3.139, + "theoretical_loss": 4.345038322483449, + "tokens_seen": 203816960 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473851554663992, + "loss": 3.1496, + "theoretical_loss": 4.34486404322803, + "tokens_seen": 203882496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738415245737211, + "loss": 3.1446, + "theoretical_loss": 4.34468983566389, + "tokens_seen": 203948032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047383149448345036, + "loss": 3.4082, + "theoretical_loss": 4.3445156997385155, + "tokens_seen": 204013568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047382146439317954, + "loss": 3.1106, + "theoretical_loss": 4.344341635399447, + "tokens_seen": 204079104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738114343029087, + "loss": 2.9751, + "theoretical_loss": 4.344167642594282, + "tokens_seen": 204144640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738014042126379, + "loss": 3.0756, + "theoretical_loss": 4.343993721270673, + "tokens_seen": 204210176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047379137412236714, + "loss": 3.1045, + "theoretical_loss": 4.343819871376326, + "tokens_seen": 204275712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047378134403209626, + "loss": 3.1058, + "theoretical_loss": 4.343646092859004, + "tokens_seen": 204341248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737713139418255, + "loss": 2.8941, + "theoretical_loss": 4.343472385666525, + "tokens_seen": 204406784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737612838515546, + "loss": 3.2849, + "theoretical_loss": 4.343298749746759, + "tokens_seen": 204472320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047375125376128386, + "loss": 3.0801, + "theoretical_loss": 4.343125185047634, + "tokens_seen": 204537856 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047374122367101304, + "loss": 3.5135, + "theoretical_loss": 4.342951691517131, + "tokens_seen": 204603392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737311935807422, + "loss": 3.2053, + "theoretical_loss": 4.342778269103286, + "tokens_seen": 204668928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047372116349047146, + "loss": 3.0911, + "theoretical_loss": 4.342604917754191, + "tokens_seen": 204734464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 123173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.030834436416626, + "objective/train/theoretical_loss": 4.342431637417991, + "objective/train/tokens_used": 225260000, + "theoretical_loss": 4.342431637417991, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737111334002006, + "loss": 3.1185, + "theoretical_loss": 4.342431637417991, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737011033099298, + "loss": 3.0803, + "theoretical_loss": 4.342258428042883, + "tokens_seen": 204865536 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473691073219659, + "loss": 3.2108, + "theoretical_loss": 4.342085289577124, + "tokens_seen": 204931072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736810431293882, + "loss": 3.2812, + "theoretical_loss": 4.341912221969021, + "tokens_seen": 204996608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047367101303911736, + "loss": 3.1871, + "theoretical_loss": 4.341739225166936, + "tokens_seen": 205062144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736609829488466, + "loss": 3.1366, + "theoretical_loss": 4.341566299119286, + "tokens_seen": 205127680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736509528585757, + "loss": 3.1494, + "theoretical_loss": 4.341393443774542, + "tokens_seen": 205193216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047364092276830496, + "loss": 3.0442, + "theoretical_loss": 4.341220659081228, + "tokens_seen": 205258752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736308926780341, + "loss": 3.1534, + "theoretical_loss": 4.341047944987921, + "tokens_seen": 205324288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736208625877633, + "loss": 3.1698, + "theoretical_loss": 4.340875301443257, + "tokens_seen": 205389824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736108324974925, + "loss": 3.1333, + "theoretical_loss": 4.340702728395918, + "tokens_seen": 205455360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736008024072217, + "loss": 3.1996, + "theoretical_loss": 4.340530225794646, + "tokens_seen": 205520896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047359077231695087, + "loss": 3.3751, + "theoretical_loss": 4.340357793588233, + "tokens_seen": 205586432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047358074222668005, + "loss": 3.3156, + "theoretical_loss": 4.340185431725527, + "tokens_seen": 205651968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047357071213640923, + "loss": 3.3006, + "theoretical_loss": 4.3400131401554285, + "tokens_seen": 205717504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047356068204613847, + "loss": 3.1734, + "theoretical_loss": 4.33984091882689, + "tokens_seen": 205783040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735506519558676, + "loss": 2.9499, + "theoretical_loss": 4.339668767688918, + "tokens_seen": 205848576 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047354062186559683, + "loss": 2.998, + "theoretical_loss": 4.339496686690575, + "tokens_seen": 205914112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047353059177532595, + "loss": 3.2018, + "theoretical_loss": 4.339324675780972, + "tokens_seen": 205979648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735205616850552, + "loss": 3.0006, + "theoretical_loss": 4.339152734909277, + "tokens_seen": 206045184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047351053159478437, + "loss": 3.1032, + "theoretical_loss": 4.338980864024709, + "tokens_seen": 206110720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047350050150451355, + "loss": 3.1907, + "theoretical_loss": 4.33880906307654, + "tokens_seen": 206176256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047349047141424273, + "loss": 3.1745, + "theoretical_loss": 4.338637332014096, + "tokens_seen": 206241792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047348044132397197, + "loss": 3.1799, + "theoretical_loss": 4.338465670786754, + "tokens_seen": 206307328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734704112337011, + "loss": 3.1274, + "theoretical_loss": 4.338294079343946, + "tokens_seen": 206372864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 123915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9498279094696045, + "objective/train/theoretical_loss": 4.338122557635156, + "objective/train/tokens_used": 226898400, + "theoretical_loss": 4.338122557635156, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047346038114343033, + "loss": 3.1472, + "theoretical_loss": 4.338122557635156, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047345035105315946, + "loss": 3.5027, + "theoretical_loss": 4.3379511056099185, + "tokens_seen": 206503936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734403209628887, + "loss": 3.3358, + "theoretical_loss": 4.337779723217823, + "tokens_seen": 206569472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734302908726179, + "loss": 3.3308, + "theoretical_loss": 4.33760841040851, + "tokens_seen": 206635008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047342026078234706, + "loss": 3.2363, + "theoretical_loss": 4.337437167131674, + "tokens_seen": 206700544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047341023069207624, + "loss": 3.339, + "theoretical_loss": 4.33726599333706, + "tokens_seen": 206766080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734002006018054, + "loss": 3.0416, + "theoretical_loss": 4.337094888974467, + "tokens_seen": 206831616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733901705115346, + "loss": 3.0678, + "theoretical_loss": 4.336923853993743, + "tokens_seen": 206897152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047338014042126383, + "loss": 3.1236, + "theoretical_loss": 4.336752888344792, + "tokens_seen": 206962688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047337011033099296, + "loss": 3.1134, + "theoretical_loss": 4.336581991977568, + "tokens_seen": 207028224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733600802407222, + "loss": 2.9821, + "theoretical_loss": 4.336411164842078, + "tokens_seen": 207093760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733500501504513, + "loss": 3.0005, + "theoretical_loss": 4.336240406888377, + "tokens_seen": 207159296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047334002006018056, + "loss": 3.0117, + "theoretical_loss": 4.336069718066579, + "tokens_seen": 207224832 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047332998996990974, + "loss": 2.9441, + "theoretical_loss": 4.335899098326843, + "tokens_seen": 207290368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733199598796389, + "loss": 3.0821, + "theoretical_loss": 4.335728547619382, + "tokens_seen": 207355904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733099297893681, + "loss": 3.1602, + "theoretical_loss": 4.335558065894464, + "tokens_seen": 207421440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047329989969909734, + "loss": 3.2261, + "theoretical_loss": 4.335387653102402, + "tokens_seen": 207486976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047328986960882646, + "loss": 3.2569, + "theoretical_loss": 4.335217309193567, + "tokens_seen": 207552512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732798395185557, + "loss": 3.0878, + "theoretical_loss": 4.335047034118375, + "tokens_seen": 207618048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732698094282848, + "loss": 3.0245, + "theoretical_loss": 4.334876827827299, + "tokens_seen": 207683584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325977933801406, + "loss": 3.0552, + "theoretical_loss": 4.33470669027086, + "tokens_seen": 207749120 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047324974924774324, + "loss": 3.0553, + "theoretical_loss": 4.334536621399632, + "tokens_seen": 207814656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732397191574724, + "loss": 3.1419, + "theoretical_loss": 4.3343666211642375, + "tokens_seen": 207880192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732296890672016, + "loss": 2.9697, + "theoretical_loss": 4.334196689515354, + "tokens_seen": 207945728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732196589769308, + "loss": 3.4403, + "theoretical_loss": 4.334026826403705, + "tokens_seen": 208011264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 125279, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2541067600250244, + "objective/train/theoretical_loss": 4.333857031780069, + "objective/train/tokens_used": 228536800, + "theoretical_loss": 4.333857031780069, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047320962888665997, + "loss": 3.1755, + "theoretical_loss": 4.333857031780069, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731995987963892, + "loss": 3.3347, + "theoretical_loss": 4.333687305595276, + "tokens_seen": 208142336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047318956870611833, + "loss": 3.1738, + "theoretical_loss": 4.3335176478002015, + "tokens_seen": 208207872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047317953861584757, + "loss": 3.0407, + "theoretical_loss": 4.333348058345776, + "tokens_seen": 208273408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047316950852557675, + "loss": 3.0055, + "theoretical_loss": 4.33317853718298, + "tokens_seen": 208338944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047315947843530593, + "loss": 3.1805, + "theoretical_loss": 4.333009084262844, + "tokens_seen": 208404480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731494483450351, + "loss": 2.9499, + "theoretical_loss": 4.33283969953645, + "tokens_seen": 208470016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731394182547643, + "loss": 3.0383, + "theoretical_loss": 4.332670382954928, + "tokens_seen": 208535552 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047312938816449347, + "loss": 3.1877, + "theoretical_loss": 4.33250113446946, + "tokens_seen": 208601088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731193580742227, + "loss": 3.1835, + "theoretical_loss": 4.332331954031279, + "tokens_seen": 208666624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047310932798395183, + "loss": 3.2001, + "theoretical_loss": 4.332162841591668, + "tokens_seen": 208732160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047309929789368107, + "loss": 3.2484, + "theoretical_loss": 4.331993797101958, + "tokens_seen": 208797696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730892678034102, + "loss": 3.37, + "theoretical_loss": 4.331824820513534, + "tokens_seen": 208863232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047307923771313943, + "loss": 3.1849, + "theoretical_loss": 4.331655911777826, + "tokens_seen": 208928768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730692076228686, + "loss": 3.219, + "theoretical_loss": 4.331487070846319, + "tokens_seen": 208994304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730591775325978, + "loss": 3.3387, + "theoretical_loss": 4.331318297670545, + "tokens_seen": 209059840 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473049147442327, + "loss": 3.1956, + "theoretical_loss": 4.3311495922020855, + "tokens_seen": 209125376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303911735205615, + "loss": 3.1449, + "theoretical_loss": 4.330980954392573, + "tokens_seen": 209190912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047302908726178534, + "loss": 2.8264, + "theoretical_loss": 4.33081238419369, + "tokens_seen": 209256448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047301905717151457, + "loss": 3.0265, + "theoretical_loss": 4.330643881557168, + "tokens_seen": 209321984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730090270812437, + "loss": 3.2387, + "theoretical_loss": 4.330475446434788, + "tokens_seen": 209387520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047299899699097293, + "loss": 3.2265, + "theoretical_loss": 4.330307078778379, + "tokens_seen": 209453056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729889669007021, + "loss": 3.2092, + "theoretical_loss": 4.330138778539823, + "tokens_seen": 209518592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729789368104313, + "loss": 3.2283, + "theoretical_loss": 4.329970545671048, + "tokens_seen": 209584128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047296890672016053, + "loss": 3.2865, + "theoretical_loss": 4.3298023801240335, + "tokens_seen": 209649664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 125881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1459879875183105, + "objective/train/theoretical_loss": 4.329634281850807, + "objective/train/tokens_used": 230175200, + "theoretical_loss": 4.329634281850807, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047295887662988966, + "loss": 3.168, + "theoretical_loss": 4.329634281850807, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729488465396189, + "loss": 3.1116, + "theoretical_loss": 4.329466250803446, + "tokens_seen": 209780736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729388164493481, + "loss": 3.1933, + "theoretical_loss": 4.329298286934075, + "tokens_seen": 209846272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047292878635907726, + "loss": 2.9711, + "theoretical_loss": 4.329130390194871, + "tokens_seen": 209911808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047291875626880644, + "loss": 3.0108, + "theoretical_loss": 4.3289625605380575, + "tokens_seen": 209977344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729087261785356, + "loss": 3.1506, + "theoretical_loss": 4.328794797915908, + "tokens_seen": 210042880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728986960882648, + "loss": 3.237, + "theoretical_loss": 4.328627102280742, + "tokens_seen": 210108416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047288866599799403, + "loss": 3.1071, + "theoretical_loss": 4.328459473584933, + "tokens_seen": 210173952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047287863590772316, + "loss": 3.2433, + "theoretical_loss": 4.328291911780899, + "tokens_seen": 210239488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728686058174524, + "loss": 3.207, + "theoretical_loss": 4.328124416821107, + "tokens_seen": 210305024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728585757271815, + "loss": 3.0205, + "theoretical_loss": 4.327956988658076, + "tokens_seen": 210370560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047284854563691076, + "loss": 3.2139, + "theoretical_loss": 4.3277896272443686, + "tokens_seen": 210436096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047283851554663994, + "loss": 3.0787, + "theoretical_loss": 4.3276223325325995, + "tokens_seen": 210501632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728284854563691, + "loss": 3.1398, + "theoretical_loss": 4.327455104475431, + "tokens_seen": 210567168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728184553660983, + "loss": 3.3631, + "theoretical_loss": 4.327287943025572, + "tokens_seen": 210632704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047280842527582754, + "loss": 2.9762, + "theoretical_loss": 4.327120848135781, + "tokens_seen": 210698240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047279839518555666, + "loss": 3.067, + "theoretical_loss": 4.326953819758867, + "tokens_seen": 210763776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727883650952859, + "loss": 3.2263, + "theoretical_loss": 4.326786857847681, + "tokens_seen": 210829312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472778335005015, + "loss": 3.3092, + "theoretical_loss": 4.326619962355128, + "tokens_seen": 210894848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047276830491474426, + "loss": 3.4328, + "theoretical_loss": 4.326453133234159, + "tokens_seen": 210960384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047275827482447344, + "loss": 3.1644, + "theoretical_loss": 4.326286370437772, + "tokens_seen": 211025920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727482447342026, + "loss": 3.1984, + "theoretical_loss": 4.326119673919013, + "tokens_seen": 211091456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727382146439318, + "loss": 3.0863, + "theoretical_loss": 4.325953043630978, + "tokens_seen": 211156992 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472728184553661, + "loss": 3.0964, + "theoretical_loss": 4.3257864795268075, + "tokens_seen": 211222528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047271815446339017, + "loss": 3.0099, + "theoretical_loss": 4.325619981559692, + "tokens_seen": 211288064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 127261, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0648367404937744, + "objective/train/theoretical_loss": 4.325453549682869, + "objective/train/tokens_used": 231813600, + "theoretical_loss": 4.325453549682869, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727081243731194, + "loss": 3.116, + "theoretical_loss": 4.325453549682869, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047269809428284853, + "loss": 3.0499, + "theoretical_loss": 4.325287183849625, + "tokens_seen": 211419136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047268806419257777, + "loss": 3.0799, + "theoretical_loss": 4.325120884013289, + "tokens_seen": 211484672 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047267803410230695, + "loss": 3.1633, + "theoretical_loss": 4.324954650127244, + "tokens_seen": 211550208 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047266800401203613, + "loss": 2.8267, + "theoretical_loss": 4.324788482144916, + "tokens_seen": 211615744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726579739217653, + "loss": 2.9392, + "theoretical_loss": 4.324622380019778, + "tokens_seen": 211681280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726479438314945, + "loss": 3.1858, + "theoretical_loss": 4.3244563437053545, + "tokens_seen": 211746816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047263791374122367, + "loss": 3.1446, + "theoretical_loss": 4.324290373155213, + "tokens_seen": 211812352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726278836509529, + "loss": 3.259, + "theoretical_loss": 4.324124468322971, + "tokens_seen": 211877888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047261785356068203, + "loss": 3.195, + "theoretical_loss": 4.32395862916229, + "tokens_seen": 211943424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047260782347041127, + "loss": 3.0971, + "theoretical_loss": 4.32379285562688, + "tokens_seen": 212008960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725977933801404, + "loss": 3.1254, + "theoretical_loss": 4.323627147670498, + "tokens_seen": 212074496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047258776328986963, + "loss": 3.0839, + "theoretical_loss": 4.323461505246949, + "tokens_seen": 212140032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725777331995988, + "loss": 2.8079, + "theoretical_loss": 4.323295928310084, + "tokens_seen": 212205568 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472567703109328, + "loss": 3.1229, + "theoretical_loss": 4.3231304168137985, + "tokens_seen": 212271104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725576730190572, + "loss": 3.1019, + "theoretical_loss": 4.322964970712039, + "tokens_seen": 212336640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047254764292878636, + "loss": 3.272, + "theoretical_loss": 4.322799589958794, + "tokens_seen": 212402176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047253761283851554, + "loss": 2.7217, + "theoretical_loss": 4.3226342745081014, + "tokens_seen": 212467712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047252758274824477, + "loss": 3.0247, + "theoretical_loss": 4.322469024314046, + "tokens_seen": 212533248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725175526579739, + "loss": 3.2237, + "theoretical_loss": 4.322303839330757, + "tokens_seen": 212598784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047250752256770313, + "loss": 3.3425, + "theoretical_loss": 4.322138719512411, + "tokens_seen": 212664320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724974924774323, + "loss": 3.2206, + "theoretical_loss": 4.321973664813231, + "tokens_seen": 212729856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724874623871615, + "loss": 3.2825, + "theoretical_loss": 4.321808675187486, + "tokens_seen": 212795392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724774322968907, + "loss": 2.8949, + "theoretical_loss": 4.321643750589493, + "tokens_seen": 212860928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047246740220661986, + "loss": 3.0597, + "theoretical_loss": 4.32147889097361, + "tokens_seen": 212926464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 127839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5590968132019043, + "objective/train/theoretical_loss": 4.321314096294248, + "objective/train/tokens_used": 233452000, + "theoretical_loss": 4.321314096294248, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047245737211634904, + "loss": 2.9514, + "theoretical_loss": 4.321314096294248, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724473420260783, + "loss": 3.0759, + "theoretical_loss": 4.321149366505858, + "tokens_seen": 213057536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724373119358074, + "loss": 2.9617, + "theoretical_loss": 4.32098470156294, + "tokens_seen": 213123072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047242728184553664, + "loss": 2.9706, + "theoretical_loss": 4.320820101420041, + "tokens_seen": 213188608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047241725175526576, + "loss": 3.3507, + "theoretical_loss": 4.320655566031751, + "tokens_seen": 213254144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472407221664995, + "loss": 3.0787, + "theoretical_loss": 4.320491095352706, + "tokens_seen": 213319680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723971915747242, + "loss": 3.2185, + "theoretical_loss": 4.320326689337589, + "tokens_seen": 213385216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047238716148445336, + "loss": 3.2852, + "theoretical_loss": 4.320162347941129, + "tokens_seen": 213450752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047237713139418254, + "loss": 3.0982, + "theoretical_loss": 4.3199980711181, + "tokens_seen": 213516288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723671013039117, + "loss": 3.1012, + "theoretical_loss": 4.319833858823321, + "tokens_seen": 213581824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723570712136409, + "loss": 2.945, + "theoretical_loss": 4.319669711011656, + "tokens_seen": 213647360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047234704112337014, + "loss": 3.1732, + "theoretical_loss": 4.319505627638016, + "tokens_seen": 213712896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047233701103309927, + "loss": 3.1215, + "theoretical_loss": 4.319341608657357, + "tokens_seen": 213778432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723269809428285, + "loss": 3.3584, + "theoretical_loss": 4.319177654024679, + "tokens_seen": 213843968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723169508525577, + "loss": 3.0733, + "theoretical_loss": 4.319013763695029, + "tokens_seen": 213909504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047230692076228686, + "loss": 3.2797, + "theoretical_loss": 4.318849937623497, + "tokens_seen": 213975040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047229689067201605, + "loss": 2.9947, + "theoretical_loss": 4.318686175765221, + "tokens_seen": 214040576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722868605817452, + "loss": 3.0984, + "theoretical_loss": 4.31852247807538, + "tokens_seen": 214106112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722768304914744, + "loss": 2.8851, + "theoretical_loss": 4.318358844509202, + "tokens_seen": 214171648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047226680040120364, + "loss": 3.0447, + "theoretical_loss": 4.318195275021959, + "tokens_seen": 214237184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047225677031093277, + "loss": 3.2885, + "theoretical_loss": 4.318031769568966, + "tokens_seen": 214302720 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472246740220662, + "loss": 3.0849, + "theoretical_loss": 4.317868328105585, + "tokens_seen": 214368256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047223671013039113, + "loss": 3.052, + "theoretical_loss": 4.3177049505872205, + "tokens_seen": 214433792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047222668004012037, + "loss": 3.0657, + "theoretical_loss": 4.317541636969324, + "tokens_seen": 214499328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722166499498496, + "loss": 2.9079, + "theoretical_loss": 4.31737838720739, + "tokens_seen": 214564864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 128852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.337009906768799, + "objective/train/theoretical_loss": 4.317215201256958, + "objective/train/tokens_used": 235090400, + "theoretical_loss": 4.317215201256958, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047220661985957873, + "loss": 3.1137, + "theoretical_loss": 4.317215201256958, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047219658976930797, + "loss": 3.0925, + "theoretical_loss": 4.317052079073613, + "tokens_seen": 214695936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047218655967903715, + "loss": 3.1037, + "theoretical_loss": 4.316889020612983, + "tokens_seen": 214761472 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047217652958876633, + "loss": 2.9058, + "theoretical_loss": 4.316726025830741, + "tokens_seen": 214827008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721664994984955, + "loss": 3.1171, + "theoretical_loss": 4.316563094682604, + "tokens_seen": 214892544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721564694082247, + "loss": 3.2248, + "theoretical_loss": 4.316400227124335, + "tokens_seen": 214958080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047214643931795387, + "loss": 3.1588, + "theoretical_loss": 4.3162374231117395, + "tokens_seen": 215023616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721364092276831, + "loss": 3.2238, + "theoretical_loss": 4.316074682600666, + "tokens_seen": 215089152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047212637913741223, + "loss": 3.3115, + "theoretical_loss": 4.315912005547011, + "tokens_seen": 215154688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047211634904714147, + "loss": 2.9668, + "theoretical_loss": 4.315749391906712, + "tokens_seen": 215220224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721063189568706, + "loss": 3.0336, + "theoretical_loss": 4.31558684163575, + "tokens_seen": 215285760 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047209628886659983, + "loss": 3.1727, + "theoretical_loss": 4.315424354690153, + "tokens_seen": 215351296 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472086258776329, + "loss": 3.0615, + "theoretical_loss": 4.31526193102599, + "tokens_seen": 215416832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720762286860582, + "loss": 2.9988, + "theoretical_loss": 4.315099570599374, + "tokens_seen": 215482368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720661985957874, + "loss": 3.0711, + "theoretical_loss": 4.314937273366466, + "tokens_seen": 215547904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047205616850551656, + "loss": 3.2474, + "theoretical_loss": 4.314775039283463, + "tokens_seen": 215613440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047204613841524574, + "loss": 3.2283, + "theoretical_loss": 4.314612868306614, + "tokens_seen": 215678976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047203610832497497, + "loss": 3.1768, + "theoretical_loss": 4.314450760392204, + "tokens_seen": 215744512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720260782347041, + "loss": 3.23, + "theoretical_loss": 4.314288715496568, + "tokens_seen": 215810048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047201604814443333, + "loss": 3.443, + "theoretical_loss": 4.314126733576082, + "tokens_seen": 215875584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720060180541625, + "loss": 2.9834, + "theoretical_loss": 4.3139648145871625, + "tokens_seen": 215941120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719959879638917, + "loss": 3.3007, + "theoretical_loss": 4.313802958486274, + "tokens_seen": 216006656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719859578736209, + "loss": 3.2257, + "theoretical_loss": 4.313641165229922, + "tokens_seen": 216072192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047197592778335006, + "loss": 2.9371, + "theoretical_loss": 4.313479434774655, + "tokens_seen": 216137728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047196589769307924, + "loss": 3.1597, + "theoretical_loss": 4.313317767077065, + "tokens_seen": 216203264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 129467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.151235342025757, + "objective/train/theoretical_loss": 4.3131561620937875, + "objective/train/tokens_used": 236728800, + "theoretical_loss": 4.3131561620937875, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719558676028085, + "loss": 3.1703, + "theoretical_loss": 4.3131561620937875, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719458375125376, + "loss": 3.1118, + "theoretical_loss": 4.312994619781502, + "tokens_seen": 216334336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193580742226684, + "loss": 3.2158, + "theoretical_loss": 4.312833140096929, + "tokens_seen": 216399872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047192577733199596, + "loss": 3.1632, + "theoretical_loss": 4.312671722996835, + "tokens_seen": 216465408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719157472417252, + "loss": 3.0144, + "theoretical_loss": 4.312510368438025, + "tokens_seen": 216530944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719057171514544, + "loss": 2.9652, + "theoretical_loss": 4.31234907637735, + "tokens_seen": 216596480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047189568706118356, + "loss": 3.0804, + "theoretical_loss": 4.312187846771704, + "tokens_seen": 216662016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047188565697091274, + "loss": 3.221, + "theoretical_loss": 4.312026679578022, + "tokens_seen": 216727552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718756268806419, + "loss": 3.2019, + "theoretical_loss": 4.311865574753283, + "tokens_seen": 216793088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718655967903711, + "loss": 2.9076, + "theoretical_loss": 4.311704532254509, + "tokens_seen": 216858624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047185556670010034, + "loss": 3.1173, + "theoretical_loss": 4.311543552038762, + "tokens_seen": 216924160 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047184553660982947, + "loss": 2.9789, + "theoretical_loss": 4.31138263406315, + "tokens_seen": 216989696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718355065195587, + "loss": 3.0793, + "theoretical_loss": 4.311221778284821, + "tokens_seen": 217055232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718254764292879, + "loss": 3.1944, + "theoretical_loss": 4.311060984660967, + "tokens_seen": 217120768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047181544633901706, + "loss": 2.9635, + "theoretical_loss": 4.310900253148821, + "tokens_seen": 217186304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047180541624874625, + "loss": 3.0845, + "theoretical_loss": 4.31073958370566, + "tokens_seen": 217251840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717953861584754, + "loss": 3.3273, + "theoretical_loss": 4.3105789762888005, + "tokens_seen": 217317376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717853560682046, + "loss": 3.1426, + "theoretical_loss": 4.310418430855604, + "tokens_seen": 217382912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177532597793384, + "loss": 3.1465, + "theoretical_loss": 4.310257947363474, + "tokens_seen": 217448448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047176529588766297, + "loss": 3.1651, + "theoretical_loss": 4.310097525769854, + "tokens_seen": 217513984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717552657973922, + "loss": 3.406, + "theoretical_loss": 4.309937166032231, + "tokens_seen": 217579520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047174523570712133, + "loss": 3.3688, + "theoretical_loss": 4.309776868108134, + "tokens_seen": 217645056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173520561685057, + "loss": 3.2583, + "theoretical_loss": 4.309616631955134, + "tokens_seen": 217710592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047172517552657975, + "loss": 3.2181, + "theoretical_loss": 4.309456457530843, + "tokens_seen": 217776128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047171514543630893, + "loss": 3.0626, + "theoretical_loss": 4.309296344792916, + "tokens_seen": 217841664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 130589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.174252510070801, + "objective/train/theoretical_loss": 4.309136293699048, + "objective/train/tokens_used": 238367200, + "theoretical_loss": 4.309136293699048, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717051153460381, + "loss": 3.0759, + "theoretical_loss": 4.309136293699048, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169508525576735, + "loss": 3.0866, + "theoretical_loss": 4.308976304206978, + "tokens_seen": 217972736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716850551654965, + "loss": 3.0519, + "theoretical_loss": 4.3088163762744855, + "tokens_seen": 218038272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716750250752257, + "loss": 2.938, + "theoretical_loss": 4.308656509859392, + "tokens_seen": 218103808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047166499498495484, + "loss": 2.9371, + "theoretical_loss": 4.308496704919559, + "tokens_seen": 218169344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047165496489468407, + "loss": 2.8471, + "theoretical_loss": 4.308336961412892, + "tokens_seen": 218234880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047164493480441325, + "loss": 3.1918, + "theoretical_loss": 4.308177279297336, + "tokens_seen": 218300416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163490471414243, + "loss": 3.0599, + "theoretical_loss": 4.308017658530879, + "tokens_seen": 218365952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716248746238716, + "loss": 3.1856, + "theoretical_loss": 4.307858099071549, + "tokens_seen": 218431488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716148445336008, + "loss": 2.7223, + "theoretical_loss": 4.307698600877416, + "tokens_seen": 218497024 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047160481444333, + "loss": 3.1367, + "theoretical_loss": 4.30753916390659, + "tokens_seen": 218562560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715947843530592, + "loss": 3.048, + "theoretical_loss": 4.307379788117224, + "tokens_seen": 218628096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047158475426278834, + "loss": 3.1323, + "theoretical_loss": 4.3072204734675115, + "tokens_seen": 218693632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715747241725176, + "loss": 2.804, + "theoretical_loss": 4.307061219915687, + "tokens_seen": 218759168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715646940822467, + "loss": 3.1227, + "theoretical_loss": 4.306902027420025, + "tokens_seen": 218824704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047155466399197594, + "loss": 3.3008, + "theoretical_loss": 4.306742895938844, + "tokens_seen": 218890240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715446339017051, + "loss": 3.1554, + "theoretical_loss": 4.306583825430499, + "tokens_seen": 218955776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715346038114343, + "loss": 2.8274, + "theoretical_loss": 4.30642481585339, + "tokens_seen": 219021312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715245737211635, + "loss": 3.1366, + "theoretical_loss": 4.306265867165955, + "tokens_seen": 219086848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715145436308927, + "loss": 2.9215, + "theoretical_loss": 4.306106979326674, + "tokens_seen": 219152384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047150451354062184, + "loss": 3.2813, + "theoretical_loss": 4.305948152294068, + "tokens_seen": 219217920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714944834503511, + "loss": 2.9276, + "theoretical_loss": 4.305789386026698, + "tokens_seen": 219283456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714844533600802, + "loss": 3.0313, + "theoretical_loss": 4.305630680483166, + "tokens_seen": 219348992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047147442326980944, + "loss": 3.2763, + "theoretical_loss": 4.305472035622114, + "tokens_seen": 219414528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714643931795387, + "loss": 3.1934, + "theoretical_loss": 4.305313451402226, + "tokens_seen": 219480064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 131942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.265974283218384, + "objective/train/theoretical_loss": 4.305154927782223, + "objective/train/tokens_used": 240005600, + "theoretical_loss": 4.305154927782223, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714543630892678, + "loss": 3.0968, + "theoretical_loss": 4.305154927782223, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047144433299899704, + "loss": 3.047, + "theoretical_loss": 4.304996464720872, + "tokens_seen": 219611136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047143430290872616, + "loss": 3.1081, + "theoretical_loss": 4.304838062176974, + "tokens_seen": 219676672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714242728184554, + "loss": 3.1186, + "theoretical_loss": 4.304679720109376, + "tokens_seen": 219742208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714142427281846, + "loss": 3.161, + "theoretical_loss": 4.304521438476961, + "tokens_seen": 219807744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047140421263791376, + "loss": 3.122, + "theoretical_loss": 4.304363217238654, + "tokens_seen": 219873280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139418254764294, + "loss": 3.1317, + "theoretical_loss": 4.304205056353421, + "tokens_seen": 219938816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713841524573721, + "loss": 3.0803, + "theoretical_loss": 4.304046955780267, + "tokens_seen": 220004352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713741223671013, + "loss": 3.1409, + "theoretical_loss": 4.303888915478236, + "tokens_seen": 220069888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047136409227683054, + "loss": 3.1707, + "theoretical_loss": 4.303730935406414, + "tokens_seen": 220135424 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135406218655967, + "loss": 2.9751, + "theoretical_loss": 4.303573015523927, + "tokens_seen": 220200960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713440320962889, + "loss": 3.177, + "theoretical_loss": 4.3034151557899385, + "tokens_seen": 220266496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713340020060181, + "loss": 3.054, + "theoretical_loss": 4.303257356163654, + "tokens_seen": 220332032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047132397191574726, + "loss": 3.4153, + "theoretical_loss": 4.303099616604318, + "tokens_seen": 220397568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047131394182547645, + "loss": 3.0444, + "theoretical_loss": 4.302941937071216, + "tokens_seen": 220463104 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047130391173520563, + "loss": 2.9421, + "theoretical_loss": 4.3027843175236695, + "tokens_seen": 220528640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712938816449348, + "loss": 3.1708, + "theoretical_loss": 4.302626757921045, + "tokens_seen": 220594176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047128385155466404, + "loss": 3.3514, + "theoretical_loss": 4.302469258222744, + "tokens_seen": 220659712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047127382146439317, + "loss": 3.2391, + "theoretical_loss": 4.30231181838821, + "tokens_seen": 220725248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712637913741224, + "loss": 3.122, + "theoretical_loss": 4.302154438376926, + "tokens_seen": 220790784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125376128385153, + "loss": 3.0711, + "theoretical_loss": 4.301997118148412, + "tokens_seen": 220856320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047124373119358077, + "loss": 3.2042, + "theoretical_loss": 4.301839857662231, + "tokens_seen": 220921856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047123370110330995, + "loss": 2.8998, + "theoretical_loss": 4.301682656877983, + "tokens_seen": 220987392 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047122367101303913, + "loss": 3.3909, + "theoretical_loss": 4.301525515755307, + "tokens_seen": 221052928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712136409227683, + "loss": 3.0117, + "theoretical_loss": 4.301368434253883, + "tokens_seen": 221118464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 132442, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3066797256469727, + "objective/train/theoretical_loss": 4.301211412333429, + "objective/train/tokens_used": 241644000, + "theoretical_loss": 4.301211412333429, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047120361083249755, + "loss": 3.1236, + "theoretical_loss": 4.301211412333429, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711935807422267, + "loss": 3.0539, + "theoretical_loss": 4.301054449953702, + "tokens_seen": 221249536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711835506519559, + "loss": 3.2493, + "theoretical_loss": 4.300897547074499, + "tokens_seen": 221315072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047117352056168504, + "loss": 3.494, + "theoretical_loss": 4.300740703655655, + "tokens_seen": 221380608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047116349047141427, + "loss": 2.8181, + "theoretical_loss": 4.300583919657045, + "tokens_seen": 221446144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115346038114345, + "loss": 3.1062, + "theoretical_loss": 4.300427195038582, + "tokens_seen": 221511680 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047114343029087263, + "loss": 3.2018, + "theoretical_loss": 4.300270529760219, + "tokens_seen": 221577216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711334002006018, + "loss": 2.9325, + "theoretical_loss": 4.300113923781946, + "tokens_seen": 221642752 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471123370110331, + "loss": 3.0023, + "theoretical_loss": 4.2999573770637936, + "tokens_seen": 221708288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711133400200602, + "loss": 3.1196, + "theoretical_loss": 4.299800889565831, + "tokens_seen": 221773824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711033099297894, + "loss": 3.0607, + "theoretical_loss": 4.299644461248165, + "tokens_seen": 221839360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047109327983951854, + "loss": 3.2191, + "theoretical_loss": 4.299488092070942, + "tokens_seen": 221904896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710832497492478, + "loss": 3.0538, + "theoretical_loss": 4.299331781994347, + "tokens_seen": 221970432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710732196589769, + "loss": 3.073, + "theoretical_loss": 4.2991755309786015, + "tokens_seen": 222035968 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047106318956870614, + "loss": 2.9409, + "theoretical_loss": 4.299019338983969, + "tokens_seen": 222101504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710531594784353, + "loss": 3.0287, + "theoretical_loss": 4.298863205970751, + "tokens_seen": 222167040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710431293881645, + "loss": 3.1599, + "theoretical_loss": 4.298707131899281, + "tokens_seen": 222232576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710330992978937, + "loss": 3.146, + "theoretical_loss": 4.298551116729941, + "tokens_seen": 222298112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710230692076229, + "loss": 3.085, + "theoretical_loss": 4.298395160423144, + "tokens_seen": 222363648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047101303911735204, + "loss": 3.09, + "theoretical_loss": 4.298239262939344, + "tokens_seen": 222429184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710030090270813, + "loss": 3.0419, + "theoretical_loss": 4.2980834242390324, + "tokens_seen": 222494720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709929789368104, + "loss": 3.2707, + "theoretical_loss": 4.29792764428274, + "tokens_seen": 222560256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047098294884653964, + "loss": 3.2489, + "theoretical_loss": 4.297771923031033, + "tokens_seen": 222625792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709729187562688, + "loss": 2.9268, + "theoretical_loss": 4.297616260444519, + "tokens_seen": 222691328 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470962888665998, + "loss": 3.1398, + "theoretical_loss": 4.297460656483842, + "tokens_seen": 222756864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 133596, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.741494655609131, + "objective/train/theoretical_loss": 4.297305111109683, + "objective/train/tokens_used": 243282400, + "theoretical_loss": 4.297305111109683, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709528585757272, + "loss": 2.7095, + "theoretical_loss": 4.297305111109683, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047094282848545636, + "loss": 2.9503, + "theoretical_loss": 4.297149624282762, + "tokens_seen": 222887936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047093279839518555, + "loss": 3.1134, + "theoretical_loss": 4.296994195963838, + "tokens_seen": 222953472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709227683049148, + "loss": 2.8174, + "theoretical_loss": 4.296838826113706, + "tokens_seen": 223019008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709127382146439, + "loss": 2.7694, + "theoretical_loss": 4.296683514693199, + "tokens_seen": 223084544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047090270812437314, + "loss": 3.1462, + "theoretical_loss": 4.296528261663189, + "tokens_seen": 223150080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047089267803410227, + "loss": 3.1611, + "theoretical_loss": 4.296373066984585, + "tokens_seen": 223215616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708826479438315, + "loss": 3.0784, + "theoretical_loss": 4.296217930618332, + "tokens_seen": 223281152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708726178535607, + "loss": 3.2461, + "theoretical_loss": 4.2960628525254165, + "tokens_seen": 223346688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047086258776328987, + "loss": 3.3592, + "theoretical_loss": 4.295907832666858, + "tokens_seen": 223412224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047085255767301905, + "loss": 3.134, + "theoretical_loss": 4.2957528710037165, + "tokens_seen": 223477760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708425275827483, + "loss": 2.793, + "theoretical_loss": 4.2955979674970886, + "tokens_seen": 223543296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708324974924774, + "loss": 3.1087, + "theoretical_loss": 4.295443122108108, + "tokens_seen": 223608832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047082246740220665, + "loss": 3.2745, + "theoretical_loss": 4.295288334797946, + "tokens_seen": 223674368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708124373119358, + "loss": 3.0884, + "theoretical_loss": 4.295133605527812, + "tokens_seen": 223739904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470802407221665, + "loss": 3.0021, + "theoretical_loss": 4.294978934258951, + "tokens_seen": 223805440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707923771313942, + "loss": 2.9157, + "theoretical_loss": 4.294824320952646, + "tokens_seen": 223870976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047078234704112337, + "loss": 3.3173, + "theoretical_loss": 4.294669765570219, + "tokens_seen": 223936512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047077231695085255, + "loss": 3.1127, + "theoretical_loss": 4.294515268073026, + "tokens_seen": 224002048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047076228686058173, + "loss": 2.8344, + "theoretical_loss": 4.294360828422461, + "tokens_seen": 224067584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707522567703109, + "loss": 3.0905, + "theoretical_loss": 4.294206446579956, + "tokens_seen": 224133120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047074222668004015, + "loss": 3.1892, + "theoretical_loss": 4.29405212250698, + "tokens_seen": 224198656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707321965897693, + "loss": 2.8706, + "theoretical_loss": 4.293897856165039, + "tokens_seen": 224264192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707221664994985, + "loss": 3.2766, + "theoretical_loss": 4.293743647515674, + "tokens_seen": 224329728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047071213640922775, + "loss": 3.1984, + "theoretical_loss": 4.293589496520464, + "tokens_seen": 224395264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 134273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.291004180908203, + "objective/train/theoretical_loss": 4.293435403141026, + "objective/train/tokens_used": 244920800, + "theoretical_loss": 4.293435403141026, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707021063189569, + "loss": 3.2134, + "theoretical_loss": 4.293435403141026, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706920762286861, + "loss": 3.107, + "theoretical_loss": 4.293281367339013, + "tokens_seen": 224526336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047068204613841524, + "loss": 3.2989, + "theoretical_loss": 4.293127389076113, + "tokens_seen": 224591872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047067201604814447, + "loss": 3.4678, + "theoretical_loss": 4.292973468314053, + "tokens_seen": 224657408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047066198595787365, + "loss": 3.0958, + "theoretical_loss": 4.292819605014596, + "tokens_seen": 224722944 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047065195586760283, + "loss": 2.9754, + "theoretical_loss": 4.292665799139541, + "tokens_seen": 224788480 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470641925777332, + "loss": 3.2233, + "theoretical_loss": 4.292512050650723, + "tokens_seen": 224854016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706318956870612, + "loss": 2.9816, + "theoretical_loss": 4.292358359510015, + "tokens_seen": 224919552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706218655967904, + "loss": 3.2302, + "theoretical_loss": 4.292204725679326, + "tokens_seen": 224985088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706118355065196, + "loss": 3.1335, + "theoretical_loss": 4.2920511491206, + "tokens_seen": 225050624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047060180541624874, + "loss": 3.1664, + "theoretical_loss": 4.2918976297958205, + "tokens_seen": 225116160 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470591775325978, + "loss": 3.0454, + "theoretical_loss": 4.2917441676670025, + "tokens_seen": 225181696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705817452357071, + "loss": 3.0739, + "theoretical_loss": 4.291590762696202, + "tokens_seen": 225247232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047057171514543634, + "loss": 3.1721, + "theoretical_loss": 4.291437414845509, + "tokens_seen": 225312768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705616850551655, + "loss": 3.1347, + "theoretical_loss": 4.291284124077049, + "tokens_seen": 225378304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705516549648947, + "loss": 2.9492, + "theoretical_loss": 4.291130890352985, + "tokens_seen": 225443840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705416248746239, + "loss": 3.1897, + "theoretical_loss": 4.290977713635515, + "tokens_seen": 225509376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705315947843531, + "loss": 3.2739, + "theoretical_loss": 4.2908245938868745, + "tokens_seen": 225574912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047052156469408224, + "loss": 3.3183, + "theoretical_loss": 4.290671531069334, + "tokens_seen": 225640448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705115346038115, + "loss": 3.1922, + "theoretical_loss": 4.290518525145199, + "tokens_seen": 225705984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705015045135406, + "loss": 3.282, + "theoretical_loss": 4.290365576076812, + "tokens_seen": 225771520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047049147442326984, + "loss": 3.2065, + "theoretical_loss": 4.290212683826551, + "tokens_seen": 225837056 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470481444332999, + "loss": 3.251, + "theoretical_loss": 4.290059848356833, + "tokens_seen": 225902592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704714142427282, + "loss": 2.9908, + "theoretical_loss": 4.289907069630104, + "tokens_seen": 225968128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704613841524574, + "loss": 3.2375, + "theoretical_loss": 4.28975434760885, + "tokens_seen": 226033664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 135613, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.803276538848877, + "objective/train/theoretical_loss": 4.2896016822555945, + "objective/train/tokens_used": 246559200, + "theoretical_loss": 4.2896016822555945, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047045135406218656, + "loss": 3.0062, + "theoretical_loss": 4.2896016822555945, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047044132397191575, + "loss": 3.1042, + "theoretical_loss": 4.289449073532892, + "tokens_seen": 226164736 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470431293881645, + "loss": 3.3344, + "theoretical_loss": 4.289296521403337, + "tokens_seen": 226230272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704212637913741, + "loss": 3.1889, + "theoretical_loss": 4.289144025829555, + "tokens_seen": 226295808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047041123370110334, + "loss": 2.9189, + "theoretical_loss": 4.288991586774211, + "tokens_seen": 226361344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047040120361083247, + "loss": 3.2327, + "theoretical_loss": 4.288839204200003, + "tokens_seen": 226426880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703911735205617, + "loss": 2.9544, + "theoretical_loss": 4.288686878069666, + "tokens_seen": 226492416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703811434302909, + "loss": 3.1703, + "theoretical_loss": 4.288534608345969, + "tokens_seen": 226557952 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047037111334002007, + "loss": 3.1722, + "theoretical_loss": 4.288382394991717, + "tokens_seen": 226623488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047036108324974925, + "loss": 3.0562, + "theoretical_loss": 4.288230237969751, + "tokens_seen": 226689024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703510531594785, + "loss": 3.074, + "theoretical_loss": 4.288078137242946, + "tokens_seen": 226754560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703410230692076, + "loss": 2.9752, + "theoretical_loss": 4.287926092774212, + "tokens_seen": 226820096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047033099297893685, + "loss": 2.8926, + "theoretical_loss": 4.287774104526496, + "tokens_seen": 226885632 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470320962888666, + "loss": 3.2959, + "theoretical_loss": 4.287622172462777, + "tokens_seen": 226951168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703109327983952, + "loss": 3.1867, + "theoretical_loss": 4.287470296546073, + "tokens_seen": 227016704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703009027081244, + "loss": 3.0586, + "theoretical_loss": 4.287318476739433, + "tokens_seen": 227082240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047029087261785357, + "loss": 3.1358, + "theoretical_loss": 4.287166713005945, + "tokens_seen": 227147776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047028084252758275, + "loss": 3.1048, + "theoretical_loss": 4.287015005308728, + "tokens_seen": 227213312 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047027081243731193, + "loss": 3.1529, + "theoretical_loss": 4.286863353610939, + "tokens_seen": 227278848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702607823470411, + "loss": 3.0266, + "theoretical_loss": 4.286711757875768, + "tokens_seen": 227344384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047025075225677035, + "loss": 3.1715, + "theoretical_loss": 4.28656021806644, + "tokens_seen": 227409920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702407221664995, + "loss": 3.0616, + "theoretical_loss": 4.286408734146216, + "tokens_seen": 227475456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702306920762287, + "loss": 3.1768, + "theoretical_loss": 4.28625730607839, + "tokens_seen": 227540992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047022066198595784, + "loss": 3.1572, + "theoretical_loss": 4.286105933826292, + "tokens_seen": 227606528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702106318956871, + "loss": 3.07, + "theoretical_loss": 4.285954617353285, + "tokens_seen": 227672064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 136262, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.876788377761841, + "objective/train/theoretical_loss": 4.28580335662277, + "objective/train/tokens_used": 248197600, + "theoretical_loss": 4.28580335662277, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047020060180541626, + "loss": 3.0746, + "theoretical_loss": 4.28580335662277, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047019057171514544, + "loss": 3.2485, + "theoretical_loss": 4.285652151598178, + "tokens_seen": 227803136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701805416248746, + "loss": 3.2203, + "theoretical_loss": 4.285501002242977, + "tokens_seen": 227868672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047017051153460385, + "loss": 3.0961, + "theoretical_loss": 4.285349908520671, + "tokens_seen": 227934208 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470160481444333, + "loss": 3.1869, + "theoretical_loss": 4.285198870394796, + "tokens_seen": 227999744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701504513540622, + "loss": 3.154, + "theoretical_loss": 4.28504788782892, + "tokens_seen": 228065280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047014042126379134, + "loss": 3.1927, + "theoretical_loss": 4.284896960786653, + "tokens_seen": 228130816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701303911735206, + "loss": 3.2737, + "theoretical_loss": 4.284746089231631, + "tokens_seen": 228196352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047012036108324976, + "loss": 3.299, + "theoretical_loss": 4.28459527312753, + "tokens_seen": 228261888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047011033099297894, + "loss": 2.9585, + "theoretical_loss": 4.284444512438057, + "tokens_seen": 228327424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701003009027081, + "loss": 3.1119, + "theoretical_loss": 4.2842938071269545, + "tokens_seen": 228392960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700902708124373, + "loss": 3.165, + "theoretical_loss": 4.284143157157999, + "tokens_seen": 228458496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700802407221665, + "loss": 3.1586, + "theoretical_loss": 4.2839925624949995, + "tokens_seen": 228524032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700702106318957, + "loss": 3.1651, + "theoretical_loss": 4.283842023101803, + "tokens_seen": 228589568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047006018054162484, + "loss": 3.2229, + "theoretical_loss": 4.283691538942286, + "tokens_seen": 228655104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700501504513541, + "loss": 3.192, + "theoretical_loss": 4.28354110998036, + "tokens_seen": 228720640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047004012036108326, + "loss": 3.03, + "theoretical_loss": 4.283390736179975, + "tokens_seen": 228786176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047003009027081244, + "loss": 3.3334, + "theoretical_loss": 4.283240417505107, + "tokens_seen": 228851712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700200601805416, + "loss": 2.9563, + "theoretical_loss": 4.2830901539197725, + "tokens_seen": 228917248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700100300902708, + "loss": 3.0645, + "theoretical_loss": 4.282939945388019, + "tokens_seen": 228982784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047, + "loss": 3.2184, + "theoretical_loss": 4.2827897918739275, + "tokens_seen": 229048320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699899699097292, + "loss": 3.3057, + "theoretical_loss": 4.282639693341613, + "tokens_seen": 229113856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046997993981945835, + "loss": 3.179, + "theoretical_loss": 4.282489649755225, + "tokens_seen": 229179392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699699097291876, + "loss": 3.0861, + "theoretical_loss": 4.282339661078946, + "tokens_seen": 229244928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046995987963891676, + "loss": 3.2011, + "theoretical_loss": 4.282189727276991, + "tokens_seen": 229310464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 137363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0329909324645996, + "objective/train/theoretical_loss": 4.282039848313611, + "objective/train/tokens_used": 249836000, + "theoretical_loss": 4.282039848313611, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046994984954864595, + "loss": 3.003, + "theoretical_loss": 4.282039848313611, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699398194583752, + "loss": 3.2598, + "theoretical_loss": 4.281890024153089, + "tokens_seen": 229441536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699297893681043, + "loss": 3.1051, + "theoretical_loss": 4.2817402547597405, + "tokens_seen": 229507072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046991975927783354, + "loss": 3.2666, + "theoretical_loss": 4.281590540097916, + "tokens_seen": 229572608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046990972918756267, + "loss": 3.1706, + "theoretical_loss": 4.281440880131999, + "tokens_seen": 229638144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698996990972919, + "loss": 2.882, + "theoretical_loss": 4.2812912748264065, + "tokens_seen": 229703680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698896690070211, + "loss": 3.2516, + "theoretical_loss": 4.281141724145588, + "tokens_seen": 229769216 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046987963891675027, + "loss": 3.1911, + "theoretical_loss": 4.280992228054027, + "tokens_seen": 229834752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046986960882647945, + "loss": 3.337, + "theoretical_loss": 4.28084278651624, + "tokens_seen": 229900288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698595787362087, + "loss": 3.1908, + "theoretical_loss": 4.280693399496776, + "tokens_seen": 229965824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698495486459378, + "loss": 3.0165, + "theoretical_loss": 4.280544066960219, + "tokens_seen": 230031360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046983951855566705, + "loss": 2.9808, + "theoretical_loss": 4.280394788871183, + "tokens_seen": 230096896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698294884653962, + "loss": 3.0817, + "theoretical_loss": 4.280245565194319, + "tokens_seen": 230162432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698194583751254, + "loss": 3.208, + "theoretical_loss": 4.280096395894308, + "tokens_seen": 230227968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698094282848546, + "loss": 3.1627, + "theoretical_loss": 4.279947280935864, + "tokens_seen": 230293504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046979939819458377, + "loss": 3.0551, + "theoretical_loss": 4.279798220283736, + "tokens_seen": 230359040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046978936810431295, + "loss": 3.0795, + "theoretical_loss": 4.279649213902704, + "tokens_seen": 230424576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046977933801404213, + "loss": 3.3708, + "theoretical_loss": 4.2795002617575815, + "tokens_seen": 230490112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697693079237713, + "loss": 3.1081, + "theoretical_loss": 4.279351363813215, + "tokens_seen": 230555648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046975927783350055, + "loss": 2.8502, + "theoretical_loss": 4.2792025200344845, + "tokens_seen": 230621184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697492477432297, + "loss": 3.1672, + "theoretical_loss": 4.279053730386301, + "tokens_seen": 230686720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697392176529589, + "loss": 3.1429, + "theoretical_loss": 4.278904994833609, + "tokens_seen": 230752256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046972918756268804, + "loss": 3.0444, + "theoretical_loss": 4.278756313341386, + "tokens_seen": 230817792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697191574724173, + "loss": 3.0223, + "theoretical_loss": 4.278607685874642, + "tokens_seen": 230883328 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046970912738214646, + "loss": 2.8679, + "theoretical_loss": 4.278459112398418, + "tokens_seen": 230948864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 138640, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.336690664291382, + "objective/train/theoretical_loss": 4.278310592877791, + "objective/train/tokens_used": 251474400, + "theoretical_loss": 4.278310592877791, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046969909729187564, + "loss": 2.9576, + "theoretical_loss": 4.278310592877791, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696890672016048, + "loss": 2.9152, + "theoretical_loss": 4.2781621272778665, + "tokens_seen": 231079936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046967903711133405, + "loss": 3.0227, + "theoretical_loss": 4.278013715563786, + "tokens_seen": 231145472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696690070210632, + "loss": 3.1015, + "theoretical_loss": 4.27786535770072, + "tokens_seen": 231211008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696589769307924, + "loss": 3.1436, + "theoretical_loss": 4.277717053653875, + "tokens_seen": 231276544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046964894684052154, + "loss": 3.424, + "theoretical_loss": 4.277568803388486, + "tokens_seen": 231342080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696389167502508, + "loss": 2.9966, + "theoretical_loss": 4.277420606869824, + "tokens_seen": 231407616 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046962888665997996, + "loss": 2.9781, + "theoretical_loss": 4.27727246406319, + "tokens_seen": 231473152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046961885656970914, + "loss": 3.153, + "theoretical_loss": 4.277124374933917, + "tokens_seen": 231538688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696088264794383, + "loss": 3.3026, + "theoretical_loss": 4.276976339447372, + "tokens_seen": 231604224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695987963891675, + "loss": 3.1475, + "theoretical_loss": 4.276828357568952, + "tokens_seen": 231669760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695887662988967, + "loss": 2.9757, + "theoretical_loss": 4.276680429264088, + "tokens_seen": 231735296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695787362086259, + "loss": 3.2429, + "theoretical_loss": 4.276532554498242, + "tokens_seen": 231800832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046956870611835505, + "loss": 2.9495, + "theoretical_loss": 4.276384733236909, + "tokens_seen": 231866368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695586760280843, + "loss": 3.2042, + "theoretical_loss": 4.276236965445612, + "tokens_seen": 231931904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046954864593781346, + "loss": 3.0181, + "theoretical_loss": 4.276089251089914, + "tokens_seen": 231997440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046953861584754264, + "loss": 3.0194, + "theoretical_loss": 4.275941590135401, + "tokens_seen": 232062976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695285857572718, + "loss": 3.1368, + "theoretical_loss": 4.275793982547698, + "tokens_seen": 232128512 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469518555667001, + "loss": 3.1842, + "theoretical_loss": 4.275646428292456, + "tokens_seen": 232194048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695085255767302, + "loss": 3.0394, + "theoretical_loss": 4.275498927335362, + "tokens_seen": 232259584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694984954864594, + "loss": 3.2062, + "theoretical_loss": 4.275351479642134, + "tokens_seen": 232325120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046948846539618855, + "loss": 3.2058, + "theoretical_loss": 4.27520408517852, + "tokens_seen": 232390656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694784353059178, + "loss": 2.9021, + "theoretical_loss": 4.275056743910301, + "tokens_seen": 232456192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694684052156469, + "loss": 3.1015, + "theoretical_loss": 4.2749094558032885, + "tokens_seen": 232521728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046945837512537615, + "loss": 3.1834, + "theoretical_loss": 4.274762220823328, + "tokens_seen": 232587264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 139202, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1069910526275635, + "objective/train/theoretical_loss": 4.274615038936293, + "objective/train/tokens_used": 253112800, + "theoretical_loss": 4.274615038936293, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694483450351053, + "loss": 3.0744, + "theoretical_loss": 4.274615038936293, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694383149448345, + "loss": 3.0827, + "theoretical_loss": 4.274467910108093, + "tokens_seen": 232718336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694282848545637, + "loss": 3.0734, + "theoretical_loss": 4.274320834304666, + "tokens_seen": 232783872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046941825476429287, + "loss": 3.1049, + "theoretical_loss": 4.27417381149198, + "tokens_seen": 232849408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046940822467402205, + "loss": 3.0458, + "theoretical_loss": 4.274026841636037, + "tokens_seen": 232914944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693981945837513, + "loss": 3.2492, + "theoretical_loss": 4.273879924702872, + "tokens_seen": 232980480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693881644934804, + "loss": 3.0141, + "theoretical_loss": 4.273733060658546, + "tokens_seen": 233046016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046937813440320965, + "loss": 2.9252, + "theoretical_loss": 4.273586249469155, + "tokens_seen": 233111552 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046936810431293883, + "loss": 3.0968, + "theoretical_loss": 4.273439491100828, + "tokens_seen": 233177088 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469358074222668, + "loss": 2.9294, + "theoretical_loss": 4.273292785519722, + "tokens_seen": 233242624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693480441323972, + "loss": 2.9721, + "theoretical_loss": 4.273146132692023, + "tokens_seen": 233308160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693380140421264, + "loss": 3.2223, + "theoretical_loss": 4.272999532583955, + "tokens_seen": 233373696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046932798395185555, + "loss": 3.1393, + "theoretical_loss": 4.272852985161768, + "tokens_seen": 233439232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693179538615848, + "loss": 3.2015, + "theoretical_loss": 4.272706490391743, + "tokens_seen": 233504768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693079237713139, + "loss": 3.0477, + "theoretical_loss": 4.272560048240195, + "tokens_seen": 233570304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046929789368104315, + "loss": 3.0628, + "theoretical_loss": 4.272413658673468, + "tokens_seen": 233635840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692878635907723, + "loss": 3.2822, + "theoretical_loss": 4.272267321657937, + "tokens_seen": 233701376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692778335005015, + "loss": 3.2682, + "theoretical_loss": 4.272121037160009, + "tokens_seen": 233766912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692678034102307, + "loss": 3.1303, + "theoretical_loss": 4.271974805146121, + "tokens_seen": 233832448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692577733199599, + "loss": 3.1491, + "theoretical_loss": 4.27182862558274, + "tokens_seen": 233897984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046924774322968906, + "loss": 3.1131, + "theoretical_loss": 4.271682498436365, + "tokens_seen": 233963520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046923771313941824, + "loss": 3.1811, + "theoretical_loss": 4.271536423673528, + "tokens_seen": 234029056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692276830491474, + "loss": 3.2045, + "theoretical_loss": 4.271390401260785, + "tokens_seen": 234094592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046921765295887666, + "loss": 3.1978, + "theoretical_loss": 4.271244431164731, + "tokens_seen": 234160128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046920762286860584, + "loss": 3.2577, + "theoretical_loss": 4.271098513351986, + "tokens_seen": 234225664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 140513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1093058586120605, + "objective/train/theoretical_loss": 4.270952647789202, + "objective/train/tokens_used": 254751200, + "theoretical_loss": 4.270952647789202, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469197592778335, + "loss": 3.3018, + "theoretical_loss": 4.270952647789202, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046918756268806425, + "loss": 3.1679, + "theoretical_loss": 4.2708068344430625, + "tokens_seen": 234356736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691775325977934, + "loss": 3.092, + "theoretical_loss": 4.270661073280282, + "tokens_seen": 234422272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691675025075226, + "loss": 3.2188, + "theoretical_loss": 4.2705153642676015, + "tokens_seen": 234487808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046915747241725174, + "loss": 3.162, + "theoretical_loss": 4.270369707371799, + "tokens_seen": 234553344 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469147442326981, + "loss": 3.123, + "theoretical_loss": 4.270224102559677, + "tokens_seen": 234618880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046913741223671016, + "loss": 3.2137, + "theoretical_loss": 4.270078549798071, + "tokens_seen": 234684416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046912738214643934, + "loss": 3.1062, + "theoretical_loss": 4.269933049053847, + "tokens_seen": 234749952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691173520561685, + "loss": 3.2631, + "theoretical_loss": 4.269787600293903, + "tokens_seen": 234815488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691073219658977, + "loss": 3.0201, + "theoretical_loss": 4.269642203485162, + "tokens_seen": 234881024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690972918756269, + "loss": 2.9759, + "theoretical_loss": 4.269496858594582, + "tokens_seen": 234946560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690872617853561, + "loss": 3.1686, + "theoretical_loss": 4.269351565589151, + "tokens_seen": 235012096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046907723169508525, + "loss": 3.1832, + "theoretical_loss": 4.269206324435885, + "tokens_seen": 235077632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690672016048145, + "loss": 3.1976, + "theoretical_loss": 4.26906113510183, + "tokens_seen": 235143168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046905717151454366, + "loss": 3.1711, + "theoretical_loss": 4.268915997554065, + "tokens_seen": 235208704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046904714142427284, + "loss": 3.105, + "theoretical_loss": 4.268770911759697, + "tokens_seen": 235274240 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469037111334002, + "loss": 2.9095, + "theoretical_loss": 4.268625877685863, + "tokens_seen": 235339776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690270812437312, + "loss": 3.0525, + "theoretical_loss": 4.268480895299731, + "tokens_seen": 235405312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690170511534604, + "loss": 3.2324, + "theoretical_loss": 4.268335964568497, + "tokens_seen": 235470848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690070210631896, + "loss": 3.1499, + "theoretical_loss": 4.26819108545939, + "tokens_seen": 235536384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046899699097291875, + "loss": 3.1792, + "theoretical_loss": 4.268046257939666, + "tokens_seen": 235601920 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468986960882648, + "loss": 3.2761, + "theoretical_loss": 4.2679014819766135, + "tokens_seen": 235667456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689769307923771, + "loss": 3.0686, + "theoretical_loss": 4.267756757537548, + "tokens_seen": 235732992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046896690070210635, + "loss": 3.2238, + "theoretical_loss": 4.267612084589816, + "tokens_seen": 235798528 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046895687061183553, + "loss": 3.346, + "theoretical_loss": 4.267467463100796, + "tokens_seen": 235864064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 141147, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.236262321472168, + "objective/train/theoretical_loss": 4.267322893037893, + "objective/train/tokens_used": 256389600, + "theoretical_loss": 4.267322893037893, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689468405215647, + "loss": 3.064, + "theoretical_loss": 4.267322893037893, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689368104312939, + "loss": 3.0627, + "theoretical_loss": 4.267178374368543, + "tokens_seen": 235995136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892678034102307, + "loss": 3.1095, + "theoretical_loss": 4.267033907060211, + "tokens_seen": 236060672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046891675025075225, + "loss": 3.0446, + "theoretical_loss": 4.266889491080392, + "tokens_seen": 236126208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689067201604815, + "loss": 3.1192, + "theoretical_loss": 4.2667451263966125, + "tokens_seen": 236191744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688966900702106, + "loss": 2.9901, + "theoretical_loss": 4.266600812976426, + "tokens_seen": 236257280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046888665997993985, + "loss": 3.1156, + "theoretical_loss": 4.2664565507874155, + "tokens_seen": 236322816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046887662988966903, + "loss": 3.1053, + "theoretical_loss": 4.266312339797195, + "tokens_seen": 236388352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688665997993982, + "loss": 3.0449, + "theoretical_loss": 4.266168179973407, + "tokens_seen": 236453888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688565697091274, + "loss": 3.2738, + "theoretical_loss": 4.266024071283725, + "tokens_seen": 236519424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688465396188566, + "loss": 2.8719, + "theoretical_loss": 4.265880013695849, + "tokens_seen": 236584960 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046883650952858575, + "loss": 3.2135, + "theoretical_loss": 4.265736007177511, + "tokens_seen": 236650496 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468826479438315, + "loss": 3.0794, + "theoretical_loss": 4.265592051696469, + "tokens_seen": 236716032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688164493480441, + "loss": 2.9949, + "theoretical_loss": 4.265448147220515, + "tokens_seen": 236781568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046880641925777335, + "loss": 2.8953, + "theoretical_loss": 4.265304293717467, + "tokens_seen": 236847104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687963891675025, + "loss": 2.9984, + "theoretical_loss": 4.2651604911551715, + "tokens_seen": 236912640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687863590772317, + "loss": 3.054, + "theoretical_loss": 4.265016739501508, + "tokens_seen": 236978176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687763289869609, + "loss": 3.052, + "theoretical_loss": 4.264873038724381, + "tokens_seen": 237043712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687662988966901, + "loss": 3.0228, + "theoretical_loss": 4.264729388791726, + "tokens_seen": 237109248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046875626880641926, + "loss": 3.0213, + "theoretical_loss": 4.264585789671508, + "tokens_seen": 237174784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046874623871614844, + "loss": 3.2672, + "theoretical_loss": 4.26444224133172, + "tokens_seen": 237240320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687362086258776, + "loss": 3.0529, + "theoretical_loss": 4.264298743740384, + "tokens_seen": 237305856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872617853560686, + "loss": 2.9217, + "theoretical_loss": 4.264155296865552, + "tokens_seen": 237371392 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468716148445336, + "loss": 3.0414, + "theoretical_loss": 4.264011900675305, + "tokens_seen": 237436928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687061183550652, + "loss": 3.0795, + "theoretical_loss": 4.26386855513775, + "tokens_seen": 237502464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 142360, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9394404888153076, + "objective/train/theoretical_loss": 4.263725260221027, + "objective/train/tokens_used": 258028000, + "theoretical_loss": 4.263725260221027, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686960882647944, + "loss": 3.1755, + "theoretical_loss": 4.263725260221027, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686860581745236, + "loss": 3.2614, + "theoretical_loss": 4.263582015893302, + "tokens_seen": 237633536 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046867602808425276, + "loss": 3.2817, + "theoretical_loss": 4.263438822122771, + "tokens_seen": 237699072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046866599799398194, + "loss": 3.1451, + "theoretical_loss": 4.263295678877658, + "tokens_seen": 237764608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686559679037111, + "loss": 3.2993, + "theoretical_loss": 4.263152586126217, + "tokens_seen": 237830144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864593781344036, + "loss": 3.1979, + "theoretical_loss": 4.263009543836728, + "tokens_seen": 237895680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686359077231695, + "loss": 3.1446, + "theoretical_loss": 4.2628665519775035, + "tokens_seen": 237961216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686258776328987, + "loss": 3.3129, + "theoretical_loss": 4.262723610516881, + "tokens_seen": 238026752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046861584754262785, + "loss": 3.0843, + "theoretical_loss": 4.26258071942323, + "tokens_seen": 238092288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686058174523571, + "loss": 3.0462, + "theoretical_loss": 4.262437878664945, + "tokens_seen": 238157824 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046859578736208626, + "loss": 3.1094, + "theoretical_loss": 4.262295088210451, + "tokens_seen": 238223360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046858575727181545, + "loss": 3.0836, + "theoretical_loss": 4.262152348028202, + "tokens_seen": 238288896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685757271815446, + "loss": 3.2288, + "theoretical_loss": 4.26200965808668, + "tokens_seen": 238354432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656970912738, + "loss": 3.2874, + "theoretical_loss": 4.261867018354394, + "tokens_seen": 238419968 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468555667001003, + "loss": 3.0589, + "theoretical_loss": 4.261724428799884, + "tokens_seen": 238485504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685456369107322, + "loss": 3.0927, + "theoretical_loss": 4.261581889391715, + "tokens_seen": 238551040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046853560682046135, + "loss": 3.2002, + "theoretical_loss": 4.2614394000984825, + "tokens_seen": 238616576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685255767301906, + "loss": 3.1648, + "theoretical_loss": 4.261296960888811, + "tokens_seen": 238682112 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046851554663991977, + "loss": 3.3789, + "theoretical_loss": 4.2611545717313515, + "tokens_seen": 238747648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046850551654964895, + "loss": 3.0458, + "theoretical_loss": 4.261012232594784, + "tokens_seen": 238813184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046849548645937813, + "loss": 2.9991, + "theoretical_loss": 4.260869943447817, + "tokens_seen": 238878720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684854563691073, + "loss": 3.0852, + "theoretical_loss": 4.260727704259187, + "tokens_seen": 238944256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684754262788365, + "loss": 3.372, + "theoretical_loss": 4.260585514997658, + "tokens_seen": 239009792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046846539618856573, + "loss": 3.0656, + "theoretical_loss": 4.260443375632022, + "tokens_seen": 239075328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684553660982949, + "loss": 3.048, + "theoretical_loss": 4.260301286131099, + "tokens_seen": 239140864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 143562, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0513010025024414, + "objective/train/theoretical_loss": 4.26015924646374, + "objective/train/tokens_used": 259666400, + "theoretical_loss": 4.26015924646374, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684453360080241, + "loss": 2.9484, + "theoretical_loss": 4.26015924646374, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046843530591775327, + "loss": 3.4032, + "theoretical_loss": 4.26001725659882, + "tokens_seen": 239271936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046842527582748245, + "loss": 3.1223, + "theoretical_loss": 4.2598753165052425, + "tokens_seen": 239337472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684152457372117, + "loss": 3.1745, + "theoretical_loss": 4.259733426151941, + "tokens_seen": 239403008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684052156469408, + "loss": 3.0693, + "theoretical_loss": 4.259591585507876, + "tokens_seen": 239468544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046839518555667005, + "loss": 3.2599, + "theoretical_loss": 4.259449794542036, + "tokens_seen": 239534080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838515546639923, + "loss": 3.051, + "theoretical_loss": 4.259308053223437, + "tokens_seen": 239599616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683751253761284, + "loss": 3.2346, + "theoretical_loss": 4.259166361521121, + "tokens_seen": 239665152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683650952858576, + "loss": 3.1637, + "theoretical_loss": 4.259024719404161, + "tokens_seen": 239730688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683550651955868, + "loss": 3.0852, + "theoretical_loss": 4.258883126841656, + "tokens_seen": 239796224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834503510531595, + "loss": 3.0005, + "theoretical_loss": 4.258741583802734, + "tokens_seen": 239861760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683350050150452, + "loss": 3.2479, + "theoretical_loss": 4.258600090256548, + "tokens_seen": 239927296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683249749247743, + "loss": 2.7133, + "theoretical_loss": 4.258458646172281, + "tokens_seen": 239992832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046831494483450355, + "loss": 3.1386, + "theoretical_loss": 4.258317251519143, + "tokens_seen": 240058368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683049147442327, + "loss": 3.2625, + "theoretical_loss": 4.258175906266371, + "tokens_seen": 240123904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682948846539619, + "loss": 3.3216, + "theoretical_loss": 4.2580346103832305, + "tokens_seen": 240189440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682848545636911, + "loss": 3.1082, + "theoretical_loss": 4.257893363839013, + "tokens_seen": 240254976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682748244734203, + "loss": 2.8933, + "theoretical_loss": 4.257752166603041, + "tokens_seen": 240320512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046826479438314946, + "loss": 3.1841, + "theoretical_loss": 4.257611018644658, + "tokens_seen": 240386048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046825476429287864, + "loss": 2.9544, + "theoretical_loss": 4.257469919933241, + "tokens_seen": 240451584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682447342026078, + "loss": 3.0391, + "theoretical_loss": 4.257328870438193, + "tokens_seen": 240517120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046823470411233706, + "loss": 2.95, + "theoretical_loss": 4.257187870128942, + "tokens_seen": 240582656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682246740220662, + "loss": 3.0321, + "theoretical_loss": 4.257046918974945, + "tokens_seen": 240648192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682146439317954, + "loss": 3.1266, + "theoretical_loss": 4.2569060169456865, + "tokens_seen": 240713728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682046138415246, + "loss": 3.1535, + "theoretical_loss": 4.256765164010678, + "tokens_seen": 240779264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 144045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4141147136688232, + "objective/train/theoretical_loss": 4.256624360139456, + "objective/train/tokens_used": 261304800, + "theoretical_loss": 4.256624360139456, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681945837512538, + "loss": 3.314, + "theoretical_loss": 4.256624360139456, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046818455366098296, + "loss": 3.0936, + "theoretical_loss": 4.25648360530159, + "tokens_seen": 240910336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046817452357071214, + "loss": 2.9071, + "theoretical_loss": 4.25634289946667, + "tokens_seen": 240975872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681644934804413, + "loss": 3.0465, + "theoretical_loss": 4.256202242604317, + "tokens_seen": 241041408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046815446339017056, + "loss": 3.2584, + "theoretical_loss": 4.256061634684178, + "tokens_seen": 241106944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681444332998997, + "loss": 3.0502, + "theoretical_loss": 4.255921075675927, + "tokens_seen": 241172480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681344032096289, + "loss": 3.1567, + "theoretical_loss": 4.255780565549264, + "tokens_seen": 241238016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046812437311935805, + "loss": 3.0096, + "theoretical_loss": 4.255640104273919, + "tokens_seen": 241303552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681143430290873, + "loss": 2.7541, + "theoretical_loss": 4.255499691819647, + "tokens_seen": 241369088 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810431293881646, + "loss": 3.1162, + "theoretical_loss": 4.255359328156229, + "tokens_seen": 241434624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046809428284854565, + "loss": 3.1105, + "theoretical_loss": 4.255219013253475, + "tokens_seen": 241500160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680842527582748, + "loss": 3.1662, + "theoretical_loss": 4.255078747081219, + "tokens_seen": 241565696 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468074222668004, + "loss": 3.0573, + "theoretical_loss": 4.254938529609326, + "tokens_seen": 241631232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680641925777332, + "loss": 3.2336, + "theoretical_loss": 4.254798360807684, + "tokens_seen": 241696768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680541624874624, + "loss": 3.1564, + "theoretical_loss": 4.254658240646211, + "tokens_seen": 241762304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046804413239719155, + "loss": 3.0097, + "theoretical_loss": 4.254518169094847, + "tokens_seen": 241827840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680341023069208, + "loss": 3.0337, + "theoretical_loss": 4.254378146123563, + "tokens_seen": 241893376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802407221664997, + "loss": 2.9549, + "theoretical_loss": 4.254238171702356, + "tokens_seen": 241958912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046801404212637915, + "loss": 2.976, + "theoretical_loss": 4.2540982458012495, + "tokens_seen": 242024448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800401203610833, + "loss": 3.1121, + "theoretical_loss": 4.253958368390292, + "tokens_seen": 242089984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679939819458375, + "loss": 3.1283, + "theoretical_loss": 4.2538185394395605, + "tokens_seen": 242155520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679839518555667, + "loss": 3.2828, + "theoretical_loss": 4.253678758919156, + "tokens_seen": 242221056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046797392176529593, + "loss": 3.1396, + "theoretical_loss": 4.25353902679921, + "tokens_seen": 242286592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046796389167502505, + "loss": 3.2791, + "theoretical_loss": 4.253399343049878, + "tokens_seen": 242352128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679538615847543, + "loss": 3.0178, + "theoretical_loss": 4.253259707641341, + "tokens_seen": 242417664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 145400, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.983632802963257, + "objective/train/theoretical_loss": 4.253120120543809, + "objective/train/tokens_used": 262943200, + "theoretical_loss": 4.253120120543809, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679438314944834, + "loss": 3.1131, + "theoretical_loss": 4.253120120543809, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046793380140421265, + "loss": 3.1157, + "theoretical_loss": 4.252980581727517, + "tokens_seen": 242548736 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046792377131394183, + "loss": 3.3695, + "theoretical_loss": 4.2528410911627255, + "tokens_seen": 242614272 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467913741223671, + "loss": 3.0132, + "theoretical_loss": 4.2527016488197225, + "tokens_seen": 242679808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679037111334002, + "loss": 3.2259, + "theoretical_loss": 4.252562254668824, + "tokens_seen": 242745344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046789368104312943, + "loss": 3.3286, + "theoretical_loss": 4.2524229086803675, + "tokens_seen": 242810880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046788365095285856, + "loss": 2.9194, + "theoretical_loss": 4.252283610824723, + "tokens_seen": 242876416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678736208625878, + "loss": 3.1186, + "theoretical_loss": 4.25214436107228, + "tokens_seen": 242941952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678635907723169, + "loss": 3.0458, + "theoretical_loss": 4.25200515939346, + "tokens_seen": 243007488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046785356068204616, + "loss": 3.1178, + "theoretical_loss": 4.251866005758707, + "tokens_seen": 243073024 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046784353059177534, + "loss": 2.9132, + "theoretical_loss": 4.251726900138494, + "tokens_seen": 243138560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678335005015045, + "loss": 3.1386, + "theoretical_loss": 4.251587842503317, + "tokens_seen": 243204096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678234704112337, + "loss": 3.1516, + "theoretical_loss": 4.2514488328237, + "tokens_seen": 243269632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678134403209629, + "loss": 3.0271, + "theoretical_loss": 4.251309871070193, + "tokens_seen": 243335168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046780341023069206, + "loss": 3.1534, + "theoretical_loss": 4.25117095721337, + "tokens_seen": 243400704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677933801404213, + "loss": 3.2002, + "theoretical_loss": 4.251032091223834, + "tokens_seen": 243466240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677833500501504, + "loss": 3.2106, + "theoretical_loss": 4.250893273072213, + "tokens_seen": 243531776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046777331995987966, + "loss": 3.0027, + "theoretical_loss": 4.250754502729161, + "tokens_seen": 243597312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677632898696088, + "loss": 3.0599, + "theoretical_loss": 4.250615780165355, + "tokens_seen": 243662848 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467753259779338, + "loss": 2.7333, + "theoretical_loss": 4.250477105351502, + "tokens_seen": 243728384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677432296890672, + "loss": 3.1958, + "theoretical_loss": 4.250338478258333, + "tokens_seen": 243793920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677331995987964, + "loss": 3.2165, + "theoretical_loss": 4.250199898856604, + "tokens_seen": 243859456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046772316950852556, + "loss": 3.0671, + "theoretical_loss": 4.250061367117097, + "tokens_seen": 243924992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677131394182548, + "loss": 3.0396, + "theoretical_loss": 4.249922883010623, + "tokens_seen": 243990528 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467703109327984, + "loss": 3.0198, + "theoretical_loss": 4.249784446508014, + "tokens_seen": 244056064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 146117, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.144657850265503, + "objective/train/theoretical_loss": 4.24964605758013, + "objective/train/tokens_used": 264581600, + "theoretical_loss": 4.24964605758013, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046769307923771316, + "loss": 3.141, + "theoretical_loss": 4.24964605758013, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046768304914744234, + "loss": 3.0374, + "theoretical_loss": 4.249507716197858, + "tokens_seen": 244187136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676730190571715, + "loss": 2.9348, + "theoretical_loss": 4.249369422332107, + "tokens_seen": 244252672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046766298896690076, + "loss": 3.2713, + "theoretical_loss": 4.249231175953814, + "tokens_seen": 244318208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676529588766299, + "loss": 3.0096, + "theoretical_loss": 4.249092977033943, + "tokens_seen": 244383744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676429287863591, + "loss": 3.0521, + "theoretical_loss": 4.248954825543479, + "tokens_seen": 244449280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046763289869608825, + "loss": 3.049, + "theoretical_loss": 4.248816721453438, + "tokens_seen": 244514816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676228686058175, + "loss": 3.0529, + "theoretical_loss": 4.248678664734857, + "tokens_seen": 244580352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046761283851554666, + "loss": 2.8181, + "theoretical_loss": 4.248540655358801, + "tokens_seen": 244645888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046760280842527585, + "loss": 2.9386, + "theoretical_loss": 4.2484026932963594, + "tokens_seen": 244711424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467592778335005, + "loss": 3.1262, + "theoretical_loss": 4.248264778518647, + "tokens_seen": 244776960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675827482447342, + "loss": 3.0133, + "theoretical_loss": 4.248126910996804, + "tokens_seen": 244842496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675727181544634, + "loss": 2.8972, + "theoretical_loss": 4.2479890907019975, + "tokens_seen": 244908032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675626880641926, + "loss": 2.9783, + "theoretical_loss": 4.2478513176054165, + "tokens_seen": 244973568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046755265797392175, + "loss": 3.045, + "theoretical_loss": 4.247713591678278, + "tokens_seen": 245039104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467542627883651, + "loss": 2.8848, + "theoretical_loss": 4.247575912891824, + "tokens_seen": 245104640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046753259779338017, + "loss": 3.0541, + "theoretical_loss": 4.247438281217321, + "tokens_seen": 245170176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046752256770310935, + "loss": 3.0006, + "theoretical_loss": 4.247300696626062, + "tokens_seen": 245235712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046751253761283853, + "loss": 3.1447, + "theoretical_loss": 4.247163159089362, + "tokens_seen": 245301248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675025075225677, + "loss": 3.204, + "theoretical_loss": 4.247025668578564, + "tokens_seen": 245366784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674924774322969, + "loss": 3.2769, + "theoretical_loss": 4.246888225065036, + "tokens_seen": 245432320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046748244734202613, + "loss": 2.7871, + "theoretical_loss": 4.24675082852017, + "tokens_seen": 245497856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046747241725175525, + "loss": 2.8847, + "theoretical_loss": 4.246613478915384, + "tokens_seen": 245563392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674623871614845, + "loss": 3.0322, + "theoretical_loss": 4.246476176222119, + "tokens_seen": 245628928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674523570712136, + "loss": 3.3512, + "theoretical_loss": 4.246338920411843, + "tokens_seen": 245694464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 147054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2560789585113525, + "objective/train/theoretical_loss": 4.24620171145605, + "objective/train/tokens_used": 266220000, + "theoretical_loss": 4.24620171145605, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046744232698094285, + "loss": 3.1654, + "theoretical_loss": 4.24620171145605, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046743229689067203, + "loss": 2.945, + "theoretical_loss": 4.246064549326255, + "tokens_seen": 245825536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674222668004012, + "loss": 2.9211, + "theoretical_loss": 4.245927433994002, + "tokens_seen": 245891072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674122367101304, + "loss": 3.1739, + "theoretical_loss": 4.245790365430857, + "tokens_seen": 245956608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046740220661985963, + "loss": 3.2878, + "theoretical_loss": 4.245653343608412, + "tokens_seen": 246022144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046739217652958876, + "loss": 2.9528, + "theoretical_loss": 4.245516368498285, + "tokens_seen": 246087680 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467382146439318, + "loss": 3.1466, + "theoretical_loss": 4.245379440072116, + "tokens_seen": 246153216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673721163490471, + "loss": 2.9806, + "theoretical_loss": 4.245242558301572, + "tokens_seen": 246218752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046736208625877636, + "loss": 2.9725, + "theoretical_loss": 4.2451057231583444, + "tokens_seen": 246284288 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046735205616850554, + "loss": 3.115, + "theoretical_loss": 4.24496893461415, + "tokens_seen": 246349824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673420260782347, + "loss": 3.1308, + "theoretical_loss": 4.244832192640725, + "tokens_seen": 246415360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673319959879639, + "loss": 3.1996, + "theoretical_loss": 4.24469549720984, + "tokens_seen": 246480896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673219658976931, + "loss": 3.2534, + "theoretical_loss": 4.244558848293281, + "tokens_seen": 246546432 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046731193580742226, + "loss": 2.9928, + "theoretical_loss": 4.244422245862863, + "tokens_seen": 246611968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673019057171515, + "loss": 3.2244, + "theoretical_loss": 4.244285689890425, + "tokens_seen": 246677504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672918756268806, + "loss": 3.103, + "theoretical_loss": 4.244149180347831, + "tokens_seen": 246743040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046728184553660986, + "loss": 3.0084, + "theoretical_loss": 4.244012717206969, + "tokens_seen": 246808576 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467271815446339, + "loss": 3.0494, + "theoretical_loss": 4.24387630043975, + "tokens_seen": 246874112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672617853560682, + "loss": 3.2686, + "theoretical_loss": 4.243739930018112, + "tokens_seen": 246939648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672517552657974, + "loss": 3.0173, + "theoretical_loss": 4.243603605914016, + "tokens_seen": 247005184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672417251755266, + "loss": 3.2096, + "theoretical_loss": 4.243467328099448, + "tokens_seen": 247070720 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046723169508525576, + "loss": 3.3022, + "theoretical_loss": 4.2433310965464175, + "tokens_seen": 247136256 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467221664994985, + "loss": 3.2016, + "theoretical_loss": 4.24319491122696, + "tokens_seen": 247201792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672116349047141, + "loss": 3.0706, + "theoretical_loss": 4.243058772113131, + "tokens_seen": 247267328 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046720160481444336, + "loss": 3.0435, + "theoretical_loss": 4.242922679177017, + "tokens_seen": 247332864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 147484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0415401458740234, + "objective/train/theoretical_loss": 4.2427866323907235, + "objective/train/tokens_used": 267858400, + "theoretical_loss": 4.2427866323907235, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671915747241725, + "loss": 3.1482, + "theoretical_loss": 4.2427866323907235, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671815446339017, + "loss": 3.0442, + "theoretical_loss": 4.242650631726383, + "tokens_seen": 247463936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671715145436309, + "loss": 3.1308, + "theoretical_loss": 4.24251467715615, + "tokens_seen": 247529472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671614844533601, + "loss": 3.0189, + "theoretical_loss": 4.2423787686522045, + "tokens_seen": 247595008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046715145436308927, + "loss": 3.2769, + "theoretical_loss": 4.2422429061867515, + "tokens_seen": 247660544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046714142427281845, + "loss": 3.1076, + "theoretical_loss": 4.242107089732018, + "tokens_seen": 247726080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046713139418254763, + "loss": 2.9247, + "theoretical_loss": 4.241971319260257, + "tokens_seen": 247791616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046712136409227686, + "loss": 3.0152, + "theoretical_loss": 4.241835594743743, + "tokens_seen": 247857152 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467111334002006, + "loss": 3.1742, + "theoretical_loss": 4.241699916154779, + "tokens_seen": 247922688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671013039117352, + "loss": 2.9643, + "theoretical_loss": 4.241564283465689, + "tokens_seen": 247988224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046709127382146435, + "loss": 3.1792, + "theoretical_loss": 4.241428696648818, + "tokens_seen": 248053760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670812437311936, + "loss": 2.9589, + "theoretical_loss": 4.241293155676542, + "tokens_seen": 248119296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046707121364092277, + "loss": 2.9542, + "theoretical_loss": 4.241157660521255, + "tokens_seen": 248184832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046706118355065195, + "loss": 3.0795, + "theoretical_loss": 4.241022211155379, + "tokens_seen": 248250368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046705115346038113, + "loss": 2.8946, + "theoretical_loss": 4.240886807551356, + "tokens_seen": 248315904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046704112337011037, + "loss": 3.1148, + "theoretical_loss": 4.240751449681655, + "tokens_seen": 248381440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670310932798395, + "loss": 3.2406, + "theoretical_loss": 4.240616137518767, + "tokens_seen": 248446976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046702106318956873, + "loss": 3.0567, + "theoretical_loss": 4.240480871035208, + "tokens_seen": 248512512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046701103309929786, + "loss": 3.0778, + "theoretical_loss": 4.240345650203517, + "tokens_seen": 248578048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670010030090271, + "loss": 3.1317, + "theoretical_loss": 4.240210474996257, + "tokens_seen": 248643584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669909729187563, + "loss": 2.8978, + "theoretical_loss": 4.240075345386014, + "tokens_seen": 248709120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046698094282848545, + "loss": 3.1967, + "theoretical_loss": 4.239940261345399, + "tokens_seen": 248774656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046697091273821464, + "loss": 3.2585, + "theoretical_loss": 4.239805222847047, + "tokens_seen": 248840192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669608826479438, + "loss": 3.2483, + "theoretical_loss": 4.239670229863615, + "tokens_seen": 248905728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046695085255767305, + "loss": 3.1135, + "theoretical_loss": 4.2395352823677825, + "tokens_seen": 248971264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 148704, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5283594131469727, + "objective/train/theoretical_loss": 4.239400380332256, + "objective/train/tokens_used": 269496800, + "theoretical_loss": 4.239400380332256, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694082246740223, + "loss": 2.9646, + "theoretical_loss": 4.239400380332256, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669307923771314, + "loss": 3.1003, + "theoretical_loss": 4.239265523729765, + "tokens_seen": 249102336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669207622868606, + "loss": 3.0558, + "theoretical_loss": 4.239130712533059, + "tokens_seen": 249167872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046691073219658983, + "loss": 3.2011, + "theoretical_loss": 4.2389959467149145, + "tokens_seen": 249233408 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046690070210631896, + "loss": 3.1159, + "theoretical_loss": 4.238861226248131, + "tokens_seen": 249298944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668906720160482, + "loss": 2.9774, + "theoretical_loss": 4.238726551105531, + "tokens_seen": 249364480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668806419257773, + "loss": 2.8515, + "theoretical_loss": 4.2385919212599585, + "tokens_seen": 249430016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046687061183550656, + "loss": 3.0703, + "theoretical_loss": 4.238457336684284, + "tokens_seen": 249495552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046686058174523574, + "loss": 3.0331, + "theoretical_loss": 4.2383227973514, + "tokens_seen": 249561088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668505516549649, + "loss": 2.9647, + "theoretical_loss": 4.238188303234223, + "tokens_seen": 249626624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668405215646941, + "loss": 3.195, + "theoretical_loss": 4.23805385430569, + "tokens_seen": 249692160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668304914744233, + "loss": 3.1549, + "theoretical_loss": 4.237919450538767, + "tokens_seen": 249757696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682046138415246, + "loss": 3.257, + "theoretical_loss": 4.237785091906437, + "tokens_seen": 249823232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668104312938817, + "loss": 3.1791, + "theoretical_loss": 4.23765077838171, + "tokens_seen": 249888768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668004012036108, + "loss": 3.0966, + "theoretical_loss": 4.237516509937617, + "tokens_seen": 249954304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046679037111334006, + "loss": 3.0634, + "theoretical_loss": 4.2373822865472155, + "tokens_seen": 250019840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667803410230692, + "loss": 3.2033, + "theoretical_loss": 4.237248108183583, + "tokens_seen": 250085376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667703109327984, + "loss": 3.0081, + "theoretical_loss": 4.237113974819821, + "tokens_seen": 250150912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667602808425276, + "loss": 2.9963, + "theoretical_loss": 4.236979886429055, + "tokens_seen": 250216448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667502507522568, + "loss": 3.0911, + "theoretical_loss": 4.236845842984433, + "tokens_seen": 250281984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046674022066198596, + "loss": 3.1609, + "theoretical_loss": 4.236711844459125, + "tokens_seen": 250347520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667301905717152, + "loss": 2.8868, + "theoretical_loss": 4.236577890826326, + "tokens_seen": 250413056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667201604814443, + "loss": 2.8415, + "theoretical_loss": 4.236443982059253, + "tokens_seen": 250478592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046671013039117356, + "loss": 3.1447, + "theoretical_loss": 4.236310118131144, + "tokens_seen": 250544128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667001003009027, + "loss": 2.6377, + "theoretical_loss": 4.236176299015265, + "tokens_seen": 250609664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 149300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2010979652404785, + "objective/train/theoretical_loss": 4.2360425246849, + "objective/train/tokens_used": 271135200, + "theoretical_loss": 4.2360425246849, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666900702106319, + "loss": 3.063, + "theoretical_loss": 4.2360425246849, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666800401203611, + "loss": 3.1872, + "theoretical_loss": 4.2359087951133585, + "tokens_seen": 250740736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666700100300903, + "loss": 3.1641, + "theoretical_loss": 4.235775110273972, + "tokens_seen": 250806272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046665997993981947, + "loss": 3.2843, + "theoretical_loss": 4.235641470140094, + "tokens_seen": 250871808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046664994984954865, + "loss": 3.0174, + "theoretical_loss": 4.235507874685103, + "tokens_seen": 250937344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046663991975927783, + "loss": 2.8743, + "theoretical_loss": 4.235374323882399, + "tokens_seen": 251002880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046662988966900706, + "loss": 3.1391, + "theoretical_loss": 4.235240817705406, + "tokens_seen": 251068416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666198595787362, + "loss": 3.0944, + "theoretical_loss": 4.235107356127567, + "tokens_seen": 251133952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046660982948846543, + "loss": 3.2513, + "theoretical_loss": 4.234973939122352, + "tokens_seen": 251199488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046659979939819455, + "loss": 2.703, + "theoretical_loss": 4.2348405666632525, + "tokens_seen": 251265024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665897693079238, + "loss": 2.8943, + "theoretical_loss": 4.234707238723782, + "tokens_seen": 251330560 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046657973921765297, + "loss": 2.98, + "theoretical_loss": 4.234573955277476, + "tokens_seen": 251396096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046656970912738215, + "loss": 3.1948, + "theoretical_loss": 4.234440716297895, + "tokens_seen": 251461632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046655967903711133, + "loss": 3.2021, + "theoretical_loss": 4.234307521758619, + "tokens_seen": 251527168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046654964894684057, + "loss": 3.0329, + "theoretical_loss": 4.234174371633253, + "tokens_seen": 251592704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665396188565697, + "loss": 3.0375, + "theoretical_loss": 4.234041265895425, + "tokens_seen": 251658240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046652958876629893, + "loss": 2.9757, + "theoretical_loss": 4.233908204518783, + "tokens_seen": 251723776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046651955867602806, + "loss": 2.956, + "theoretical_loss": 4.233775187476998, + "tokens_seen": 251789312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665095285857573, + "loss": 3.0873, + "theoretical_loss": 4.233642214743766, + "tokens_seen": 251854848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664994984954865, + "loss": 3.1756, + "theoretical_loss": 4.2335092862928025, + "tokens_seen": 251920384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046648946840521565, + "loss": 3.047, + "theoretical_loss": 4.233376402097846, + "tokens_seen": 251985920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046647943831494484, + "loss": 2.7, + "theoretical_loss": 4.233243562132661, + "tokens_seen": 252051456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466469408224674, + "loss": 2.8238, + "theoretical_loss": 4.233110766371029, + "tokens_seen": 252116992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664593781344032, + "loss": 3.0103, + "theoretical_loss": 4.232978014786756, + "tokens_seen": 252182528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046644934804413243, + "loss": 2.9662, + "theoretical_loss": 4.2328453073536725, + "tokens_seen": 252248064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 150610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.978238582611084, + "objective/train/theoretical_loss": 4.232712644045627, + "objective/train/tokens_used": 272773600, + "theoretical_loss": 4.232712644045627, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046643931795386156, + "loss": 3.1531, + "theoretical_loss": 4.232712644045627, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664292878635908, + "loss": 3.2135, + "theoretical_loss": 4.232580024836494, + "tokens_seen": 252379136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664192577733199, + "loss": 3.0233, + "theoretical_loss": 4.232447449700169, + "tokens_seen": 252444672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046640922768304916, + "loss": 2.8671, + "theoretical_loss": 4.23231491861057, + "tokens_seen": 252510208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046639919759277834, + "loss": 3.0974, + "theoretical_loss": 4.232182431541636, + "tokens_seen": 252575744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663891675025075, + "loss": 3.0586, + "theoretical_loss": 4.232049988467328, + "tokens_seen": 252641280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663791374122367, + "loss": 2.9412, + "theoretical_loss": 4.2319175893616325, + "tokens_seen": 252706816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046636910732196594, + "loss": 2.9511, + "theoretical_loss": 4.231785234198555, + "tokens_seen": 252772352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046635907723169506, + "loss": 2.9213, + "theoretical_loss": 4.231652922952122, + "tokens_seen": 252837888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663490471414243, + "loss": 2.9723, + "theoretical_loss": 4.231520655596388, + "tokens_seen": 252903424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663390170511534, + "loss": 3.0252, + "theoretical_loss": 4.231388432105421, + "tokens_seen": 252968960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046632898696088266, + "loss": 3.1199, + "theoretical_loss": 4.231256252453317, + "tokens_seen": 253034496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046631895687061184, + "loss": 3.194, + "theoretical_loss": 4.231124116614194, + "tokens_seen": 253100032 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466308926780341, + "loss": 3.1208, + "theoretical_loss": 4.230992024562189, + "tokens_seen": 253165568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662988966900702, + "loss": 2.9426, + "theoretical_loss": 4.230859976271464, + "tokens_seen": 253231104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662888665997994, + "loss": 3.0458, + "theoretical_loss": 4.2307279717162, + "tokens_seen": 253296640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046627883650952857, + "loss": 3.193, + "theoretical_loss": 4.2305960108706016, + "tokens_seen": 253362176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662688064192578, + "loss": 3.1534, + "theoretical_loss": 4.230464093708894, + "tokens_seen": 253427712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046625877632898693, + "loss": 3.1943, + "theoretical_loss": 4.230332220205328, + "tokens_seen": 253493248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046624874623871616, + "loss": 3.0607, + "theoretical_loss": 4.230200390334172, + "tokens_seen": 253558784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046623871614844535, + "loss": 2.8528, + "theoretical_loss": 4.230068604069717, + "tokens_seen": 253624320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662286860581745, + "loss": 2.9088, + "theoretical_loss": 4.2299368613862764, + "tokens_seen": 253689856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662186559679037, + "loss": 3.2573, + "theoretical_loss": 4.229805162258187, + "tokens_seen": 253755392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662086258776329, + "loss": 2.9836, + "theoretical_loss": 4.229673506659805, + "tokens_seen": 253820928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661985957873621, + "loss": 3.1131, + "theoretical_loss": 4.229541894565508, + "tokens_seen": 253886464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 151678, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8198091983795166, + "objective/train/theoretical_loss": 4.229410325949698, + "objective/train/tokens_used": 274412000, + "theoretical_loss": 4.229410325949698, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661885656970913, + "loss": 2.9257, + "theoretical_loss": 4.229410325949698, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661785356068205, + "loss": 3.1713, + "theoretical_loss": 4.229278800786796, + "tokens_seen": 254017536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046616850551654967, + "loss": 2.9401, + "theoretical_loss": 4.229147319051247, + "tokens_seen": 254083072 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046615847542627885, + "loss": 2.8864, + "theoretical_loss": 4.2290158807175136, + "tokens_seen": 254148608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046614844533600803, + "loss": 3.1424, + "theoretical_loss": 4.228884485760086, + "tokens_seen": 254214144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046613841524573727, + "loss": 3.2015, + "theoretical_loss": 4.22875313415347, + "tokens_seen": 254279680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661283851554664, + "loss": 3.1367, + "theoretical_loss": 4.228621825872198, + "tokens_seen": 254345216 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046611835506519563, + "loss": 2.9269, + "theoretical_loss": 4.228490560890819, + "tokens_seen": 254410752 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046610832497492475, + "loss": 3.1191, + "theoretical_loss": 4.228359339183908, + "tokens_seen": 254476288 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466098294884654, + "loss": 3.2313, + "theoretical_loss": 4.228228160726059, + "tokens_seen": 254541824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046608826479438317, + "loss": 3.2163, + "theoretical_loss": 4.228097025491887, + "tokens_seen": 254607360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046607823470411235, + "loss": 3.0566, + "theoretical_loss": 4.22796593345603, + "tokens_seen": 254672896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046606820461384153, + "loss": 2.9742, + "theoretical_loss": 4.227834884593148, + "tokens_seen": 254738432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046605817452357077, + "loss": 2.6936, + "theoretical_loss": 4.22770387887792, + "tokens_seen": 254803968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660481444332999, + "loss": 2.989, + "theoretical_loss": 4.227572916285046, + "tokens_seen": 254869504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046603811434302913, + "loss": 3.1684, + "theoretical_loss": 4.227441996789253, + "tokens_seen": 254935040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046602808425275826, + "loss": 3.0604, + "theoretical_loss": 4.227311120365281, + "tokens_seen": 255000576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660180541624875, + "loss": 3.3224, + "theoretical_loss": 4.227180286987897, + "tokens_seen": 255066112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660080240722167, + "loss": 3.2097, + "theoretical_loss": 4.227049496631889, + "tokens_seen": 255131648 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046599799398194586, + "loss": 3.1686, + "theoretical_loss": 4.226918749272063, + "tokens_seen": 255197184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046598796389167504, + "loss": 3.0575, + "theoretical_loss": 4.226788044883248, + "tokens_seen": 255262720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659779338014042, + "loss": 3.1225, + "theoretical_loss": 4.226657383440297, + "tokens_seen": 255328256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659679037111334, + "loss": 2.9431, + "theoretical_loss": 4.226526764918079, + "tokens_seen": 255393792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595787362086263, + "loss": 3.1373, + "theoretical_loss": 4.2263961892914885, + "tokens_seen": 255459328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046594784353059176, + "loss": 3.2497, + "theoretical_loss": 4.226265656535437, + "tokens_seen": 255524864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 152377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.094141960144043, + "objective/train/theoretical_loss": 4.226135166624862, + "objective/train/tokens_used": 276050400, + "theoretical_loss": 4.226135166624862, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465937813440321, + "loss": 3.2235, + "theoretical_loss": 4.226135166624862, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659277833500501, + "loss": 3.158, + "theoretical_loss": 4.226004719534718, + "tokens_seen": 255655936 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591775325977936, + "loss": 2.9932, + "theoretical_loss": 4.225874315239982, + "tokens_seen": 255721472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046590772316950854, + "loss": 2.9383, + "theoretical_loss": 4.225743953715653, + "tokens_seen": 255787008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658976930792377, + "loss": 3.0384, + "theoretical_loss": 4.225613634936749, + "tokens_seen": 255852544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658876629889669, + "loss": 3.1105, + "theoretical_loss": 4.22548335887831, + "tokens_seen": 255918080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046587763289869614, + "loss": 3.0846, + "theoretical_loss": 4.225353125515398, + "tokens_seen": 255983616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046586760280842526, + "loss": 3.013, + "theoretical_loss": 4.225222934823094, + "tokens_seen": 256049152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658575727181545, + "loss": 3.1509, + "theoretical_loss": 4.225092786776502, + "tokens_seen": 256114688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658475426278836, + "loss": 2.9779, + "theoretical_loss": 4.224962681350746, + "tokens_seen": 256180224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046583751253761286, + "loss": 2.778, + "theoretical_loss": 4.224832618520969, + "tokens_seen": 256245760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046582748244734204, + "loss": 3.263, + "theoretical_loss": 4.224702598262336, + "tokens_seen": 256311296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658174523570712, + "loss": 2.9598, + "theoretical_loss": 4.2245726205500365, + "tokens_seen": 256376832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658074222668004, + "loss": 3.1954, + "theoretical_loss": 4.224442685359275, + "tokens_seen": 256442368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657973921765296, + "loss": 2.9688, + "theoretical_loss": 4.22431279266528, + "tokens_seen": 256507904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046578736208625877, + "loss": 3.1653, + "theoretical_loss": 4.2241829424433, + "tokens_seen": 256573440 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465777331995988, + "loss": 3.0042, + "theoretical_loss": 4.224053134668605, + "tokens_seen": 256638976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046576730190571713, + "loss": 2.9824, + "theoretical_loss": 4.223923369316485, + "tokens_seen": 256704512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575727181544636, + "loss": 2.9607, + "theoretical_loss": 4.2237936463622505, + "tokens_seen": 256770048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046574724172517555, + "loss": 3.0764, + "theoretical_loss": 4.223663965781233, + "tokens_seen": 256835584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657372116349047, + "loss": 3.1061, + "theoretical_loss": 4.2235343275487836, + "tokens_seen": 256901120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657271815446339, + "loss": 2.9893, + "theoretical_loss": 4.223404731640277, + "tokens_seen": 256966656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171514543631, + "loss": 2.9472, + "theoretical_loss": 4.223275178031105, + "tokens_seen": 257032192 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046570712136409227, + "loss": 3.0962, + "theoretical_loss": 4.223145666696682, + "tokens_seen": 257097728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656970912738215, + "loss": 3.2783, + "theoretical_loss": 4.223016197612442, + "tokens_seen": 257163264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 153004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8444950580596924, + "objective/train/theoretical_loss": 4.222886770753841, + "objective/train/tokens_used": 277688800, + "theoretical_loss": 4.222886770753841, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046568706118355063, + "loss": 2.8758, + "theoretical_loss": 4.222886770753841, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046567703109327987, + "loss": 3.2012, + "theoretical_loss": 4.222757386096354, + "tokens_seen": 257294336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465667001003009, + "loss": 3.0649, + "theoretical_loss": 4.222628043615476, + "tokens_seen": 257359872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565697091273823, + "loss": 3.1071, + "theoretical_loss": 4.2224987432867245, + "tokens_seen": 257425408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656469408224674, + "loss": 3.1137, + "theoretical_loss": 4.222369485085636, + "tokens_seen": 257490944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656369107321966, + "loss": 3.1091, + "theoretical_loss": 4.222240268987768, + "tokens_seen": 257556480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656268806419258, + "loss": 2.8566, + "theoretical_loss": 4.222111094968698, + "tokens_seen": 257622016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046561685055165495, + "loss": 3.0556, + "theoretical_loss": 4.2219819630040245, + "tokens_seen": 257687552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046560682046138414, + "loss": 3.1304, + "theoretical_loss": 4.2218528730693645, + "tokens_seen": 257753088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046559679037111337, + "loss": 2.937, + "theoretical_loss": 4.221723825140359, + "tokens_seen": 257818624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655867602808425, + "loss": 2.9502, + "theoretical_loss": 4.221594819192665, + "tokens_seen": 257884160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046557673019057173, + "loss": 3.1884, + "theoretical_loss": 4.221465855201963, + "tokens_seen": 257949696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655667001003009, + "loss": 3.1707, + "theoretical_loss": 4.221336933143952, + "tokens_seen": 258015232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655566700100301, + "loss": 2.9765, + "theoretical_loss": 4.221208052994352, + "tokens_seen": 258080768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655466399197593, + "loss": 3.1372, + "theoretical_loss": 4.221079214728903, + "tokens_seen": 258146304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553660982948846, + "loss": 3.0284, + "theoretical_loss": 4.2209504183233655, + "tokens_seen": 258211840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046552657973921764, + "loss": 3.1995, + "theoretical_loss": 4.22082166375352, + "tokens_seen": 258277376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655165496489469, + "loss": 2.8978, + "theoretical_loss": 4.220692950995167, + "tokens_seen": 258342912 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465506519558676, + "loss": 3.0251, + "theoretical_loss": 4.220564280024126, + "tokens_seen": 258408448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046549648946840524, + "loss": 2.9952, + "theoretical_loss": 4.22043565081624, + "tokens_seen": 258473984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046548645937813436, + "loss": 3.1847, + "theoretical_loss": 4.220307063347369, + "tokens_seen": 258539520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654764292878636, + "loss": 2.9241, + "theoretical_loss": 4.220178517593393, + "tokens_seen": 258605056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654663991975928, + "loss": 2.8651, + "theoretical_loss": 4.220050013530215, + "tokens_seen": 258670592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046545636910732196, + "loss": 2.9641, + "theoretical_loss": 4.219921551133754, + "tokens_seen": 258736128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654463390170512, + "loss": 3.1291, + "theoretical_loss": 4.219793130379952, + "tokens_seen": 258801664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 154245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.155519485473633, + "objective/train/theoretical_loss": 4.21966475124477, + "objective/train/tokens_used": 279327200, + "theoretical_loss": 4.21966475124477, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654363089267803, + "loss": 3.232, + "theoretical_loss": 4.21966475124477, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046542627883650956, + "loss": 3.1265, + "theoretical_loss": 4.21953641370419, + "tokens_seen": 258932736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046541624874623874, + "loss": 2.8509, + "theoretical_loss": 4.21940811773421, + "tokens_seen": 258998272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654062186559679, + "loss": 3.0692, + "theoretical_loss": 4.219279863310855, + "tokens_seen": 259063808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653961885656971, + "loss": 2.9505, + "theoretical_loss": 4.219151650410162, + "tokens_seen": 259129344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046538615847542634, + "loss": 3.049, + "theoretical_loss": 4.219023479008193, + "tokens_seen": 259194880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046537612838515546, + "loss": 3.0422, + "theoretical_loss": 4.21889534908103, + "tokens_seen": 259260416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653660982948847, + "loss": 3.2427, + "theoretical_loss": 4.218767260604771, + "tokens_seen": 259325952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653560682046138, + "loss": 2.8318, + "theoretical_loss": 4.218639213555537, + "tokens_seen": 259391488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046534603811434306, + "loss": 2.817, + "theoretical_loss": 4.218511207909467, + "tokens_seen": 259457024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533600802407224, + "loss": 2.9972, + "theoretical_loss": 4.218383243642723, + "tokens_seen": 259522560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653259779338014, + "loss": 2.8747, + "theoretical_loss": 4.2182553207314815, + "tokens_seen": 259588096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653159478435306, + "loss": 3.0103, + "theoretical_loss": 4.218127439151944, + "tokens_seen": 259653632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653059177532598, + "loss": 3.1678, + "theoretical_loss": 4.217999598880327, + "tokens_seen": 259719168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046529588766298897, + "loss": 3.1776, + "theoretical_loss": 4.2178717998928725, + "tokens_seen": 259784704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652858575727182, + "loss": 3.0578, + "theoretical_loss": 4.217744042165835, + "tokens_seen": 259850240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046527582748244733, + "loss": 3.0386, + "theoretical_loss": 4.217616325675493, + "tokens_seen": 259915776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046526579739217656, + "loss": 2.9572, + "theoretical_loss": 4.2174886503981455, + "tokens_seen": 259981312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525576730190575, + "loss": 2.985, + "theoretical_loss": 4.217361016310109, + "tokens_seen": 260046848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652457372116349, + "loss": 3.0667, + "theoretical_loss": 4.217233423387719, + "tokens_seen": 260112384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652357071213641, + "loss": 3.0605, + "theoretical_loss": 4.217105871607333, + "tokens_seen": 260177920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652256770310933, + "loss": 3.354, + "theoretical_loss": 4.216978360945325, + "tokens_seen": 260243456 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046521564694082247, + "loss": 2.9401, + "theoretical_loss": 4.216850891378092, + "tokens_seen": 260308992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652056168505517, + "loss": 3.0811, + "theoretical_loss": 4.216723462882047, + "tokens_seen": 260374528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519558676028083, + "loss": 3.1081, + "theoretical_loss": 4.2165960754336265, + "tokens_seen": 260440064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 155441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7761213779449463, + "objective/train/theoretical_loss": 4.216468729009281, + "objective/train/tokens_used": 280965600, + "theoretical_loss": 4.216468729009281, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046518555667001007, + "loss": 2.6967, + "theoretical_loss": 4.216468729009281, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651755265797392, + "loss": 3.038, + "theoretical_loss": 4.216341423585486, + "tokens_seen": 260571136 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046516549648946843, + "loss": 3.3396, + "theoretical_loss": 4.216214159138732, + "tokens_seen": 260636672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651554663991976, + "loss": 3.0875, + "theoretical_loss": 4.216086935645532, + "tokens_seen": 260702208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651454363089268, + "loss": 3.1269, + "theoretical_loss": 4.2159597530824175, + "tokens_seen": 260767744 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465135406218656, + "loss": 2.9654, + "theoretical_loss": 4.215832611425937, + "tokens_seen": 260833280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046512537612838515, + "loss": 2.9985, + "theoretical_loss": 4.215705510652662, + "tokens_seen": 260898816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046511534603811434, + "loss": 3.0659, + "theoretical_loss": 4.215578450739182, + "tokens_seen": 260964352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046510531594784357, + "loss": 2.859, + "theoretical_loss": 4.215451431662104, + "tokens_seen": 261029888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650952858575727, + "loss": 3.0105, + "theoretical_loss": 4.215324453398057, + "tokens_seen": 261095424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046508525576730193, + "loss": 3.0586, + "theoretical_loss": 4.215197515923687, + "tokens_seen": 261160960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650752256770311, + "loss": 3.0384, + "theoretical_loss": 4.21507061921566, + "tokens_seen": 261226496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650651955867603, + "loss": 2.9461, + "theoretical_loss": 4.214943763250662, + "tokens_seen": 261292032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650551654964895, + "loss": 3.0812, + "theoretical_loss": 4.214816948005396, + "tokens_seen": 261357568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046504513540621866, + "loss": 2.9739, + "theoretical_loss": 4.214690173456588, + "tokens_seen": 261423104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046503510531594784, + "loss": 3.2378, + "theoretical_loss": 4.214563439580979, + "tokens_seen": 261488640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650250752256771, + "loss": 2.9873, + "theoretical_loss": 4.214436746355331, + "tokens_seen": 261554176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650150451354062, + "loss": 3.0108, + "theoretical_loss": 4.214310093756427, + "tokens_seen": 261619712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046500501504513544, + "loss": 2.9422, + "theoretical_loss": 4.214183481761064, + "tokens_seen": 261685248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046499498495486456, + "loss": 3.1189, + "theoretical_loss": 4.214056910346063, + "tokens_seen": 261750784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649849548645938, + "loss": 3.1657, + "theoretical_loss": 4.2139303794882625, + "tokens_seen": 261816320 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464974924774323, + "loss": 2.7213, + "theoretical_loss": 4.213803889164518, + "tokens_seen": 261881856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046496489468405216, + "loss": 2.8726, + "theoretical_loss": 4.213677439351708, + "tokens_seen": 261947392 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046495486459378134, + "loss": 2.7336, + "theoretical_loss": 4.213551030026725, + "tokens_seen": 262012928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649448345035105, + "loss": 2.9864, + "theoretical_loss": 4.213424661166485, + "tokens_seen": 262078464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 156138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.797309398651123, + "objective/train/theoretical_loss": 4.21329833274792, + "objective/train/tokens_used": 282604000, + "theoretical_loss": 4.21329833274792, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649348044132397, + "loss": 2.5665, + "theoretical_loss": 4.21329833274792, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046492477432296894, + "loss": 3.0384, + "theoretical_loss": 4.213172044747982, + "tokens_seen": 262209536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046491474423269807, + "loss": 2.8508, + "theoretical_loss": 4.213045797143643, + "tokens_seen": 262275072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649047141424273, + "loss": 2.933, + "theoretical_loss": 4.212919589911891, + "tokens_seen": 262340608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648946840521565, + "loss": 2.8909, + "theoretical_loss": 4.212793423029735, + "tokens_seen": 262406144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046488465396188566, + "loss": 3.0675, + "theoretical_loss": 4.212667296474204, + "tokens_seen": 262471680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046487462387161485, + "loss": 3.0812, + "theoretical_loss": 4.2125412102223425, + "tokens_seen": 262537216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464864593781344, + "loss": 3.2516, + "theoretical_loss": 4.212415164251215, + "tokens_seen": 262602752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648545636910732, + "loss": 2.9727, + "theoretical_loss": 4.212289158537908, + "tokens_seen": 262668288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046484453360080244, + "loss": 3.2213, + "theoretical_loss": 4.212163193059521, + "tokens_seen": 262733824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046483450351053157, + "loss": 3.0169, + "theoretical_loss": 4.2120372677931766, + "tokens_seen": 262799360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648244734202608, + "loss": 3.2831, + "theoretical_loss": 4.211911382716014, + "tokens_seen": 262864896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046481444332998993, + "loss": 2.9297, + "theoretical_loss": 4.211785537805193, + "tokens_seen": 262930432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046480441323971917, + "loss": 3.0726, + "theoretical_loss": 4.211659733037891, + "tokens_seen": 262995968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046479438314944835, + "loss": 3.0694, + "theoretical_loss": 4.2115339683913025, + "tokens_seen": 263061504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046478435305917753, + "loss": 2.8509, + "theoretical_loss": 4.211408243842643, + "tokens_seen": 263127040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647743229689067, + "loss": 3.0094, + "theoretical_loss": 4.2112825593691445, + "tokens_seen": 263192576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046476429287863595, + "loss": 3.1927, + "theoretical_loss": 4.21115691494806, + "tokens_seen": 263258112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647542627883651, + "loss": 3.0788, + "theoretical_loss": 4.211031310556661, + "tokens_seen": 263323648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647442326980943, + "loss": 2.9797, + "theoretical_loss": 4.210905746172235, + "tokens_seen": 263389184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046473420260782344, + "loss": 2.7703, + "theoretical_loss": 4.210780221772088, + "tokens_seen": 263454720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046472417251755267, + "loss": 2.9913, + "theoretical_loss": 4.210654737333549, + "tokens_seen": 263520256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647141424272819, + "loss": 3.0574, + "theoretical_loss": 4.21052929283396, + "tokens_seen": 263585792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046470411233701103, + "loss": 2.9517, + "theoretical_loss": 4.210403888250685, + "tokens_seen": 263651328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046469408224674027, + "loss": 2.9469, + "theoretical_loss": 4.210278523561106, + "tokens_seen": 263716864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 157246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.921288251876831, + "objective/train/theoretical_loss": 4.2101531987426215, + "objective/train/tokens_used": 284242400, + "theoretical_loss": 4.2101531987426215, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646840521564694, + "loss": 3.1574, + "theoretical_loss": 4.2101531987426215, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046467402206619863, + "loss": 2.9865, + "theoretical_loss": 4.210027913772651, + "tokens_seen": 263847936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646639919759278, + "loss": 2.9807, + "theoretical_loss": 4.209902668628629, + "tokens_seen": 263913472 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464653961885657, + "loss": 3.0513, + "theoretical_loss": 4.209777463288012, + "tokens_seen": 263979008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646439317953862, + "loss": 2.8659, + "theoretical_loss": 4.209652297728274, + "tokens_seen": 264044544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046463390170511535, + "loss": 3.1063, + "theoretical_loss": 4.209527171926906, + "tokens_seen": 264110080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046462387161484454, + "loss": 2.9362, + "theoretical_loss": 4.209402085861418, + "tokens_seen": 264175616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046461384152457377, + "loss": 3.1464, + "theoretical_loss": 4.209277039509338, + "tokens_seen": 264241152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646038114343029, + "loss": 3.0352, + "theoretical_loss": 4.209152032848213, + "tokens_seen": 264306688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046459378134403213, + "loss": 2.7659, + "theoretical_loss": 4.209027065855607, + "tokens_seen": 264372224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645837512537613, + "loss": 2.8415, + "theoretical_loss": 4.208902138509105, + "tokens_seen": 264437760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645737211634905, + "loss": 3.2915, + "theoretical_loss": 4.208777250786305, + "tokens_seen": 264503296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645636910732197, + "loss": 3.0093, + "theoretical_loss": 4.20865240266483, + "tokens_seen": 264568832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046455366098294886, + "loss": 3.0984, + "theoretical_loss": 4.208527594122316, + "tokens_seen": 264634368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046454363089267804, + "loss": 3.171, + "theoretical_loss": 4.208402825136419, + "tokens_seen": 264699904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645336008024073, + "loss": 3.2183, + "theoretical_loss": 4.208278095684813, + "tokens_seen": 264765440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645235707121364, + "loss": 3.1366, + "theoretical_loss": 4.20815340574519, + "tokens_seen": 264830976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046451354062186564, + "loss": 2.9705, + "theoretical_loss": 4.20802875529526, + "tokens_seen": 264896512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046450351053159476, + "loss": 3.2497, + "theoretical_loss": 4.207904144312752, + "tokens_seen": 264962048 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464493480441324, + "loss": 3.1657, + "theoretical_loss": 4.2077795727754115, + "tokens_seen": 265027584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644834503510532, + "loss": 2.869, + "theoretical_loss": 4.207655040661004, + "tokens_seen": 265093120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046447342026078236, + "loss": 2.8442, + "theoretical_loss": 4.207530547947311, + "tokens_seen": 265158656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046446339017051154, + "loss": 3.1521, + "theoretical_loss": 4.207406094612133, + "tokens_seen": 265224192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644533600802407, + "loss": 2.845, + "theoretical_loss": 4.207281680633289, + "tokens_seen": 265289728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644433299899699, + "loss": 2.7843, + "theoretical_loss": 4.207157305988615, + "tokens_seen": 265355264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 157714, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.635972738265991, + "objective/train/theoretical_loss": 4.207032970655965, + "objective/train/tokens_used": 285880800, + "theoretical_loss": 4.207032970655965, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046443329989969914, + "loss": 2.9511, + "theoretical_loss": 4.207032970655965, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046442326980942827, + "loss": 3.0488, + "theoretical_loss": 4.206908674613213, + "tokens_seen": 265486336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644132397191575, + "loss": 2.7703, + "theoretical_loss": 4.2067844178382465, + "tokens_seen": 265551872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644032096288867, + "loss": 2.9723, + "theoretical_loss": 4.206660200308975, + "tokens_seen": 265617408 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046439317953861586, + "loss": 2.896, + "theoretical_loss": 4.206536022003325, + "tokens_seen": 265682944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046438314944834505, + "loss": 3.1176, + "theoretical_loss": 4.206411882899239, + "tokens_seen": 265748480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643731193580742, + "loss": 3.054, + "theoretical_loss": 4.20628778297468, + "tokens_seen": 265814016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643630892678034, + "loss": 3.0583, + "theoretical_loss": 4.206163722207627, + "tokens_seen": 265879552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046435305917753264, + "loss": 2.9522, + "theoretical_loss": 4.206039700576078, + "tokens_seen": 265945088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046434302908726177, + "loss": 3.1155, + "theoretical_loss": 4.205915718058046, + "tokens_seen": 266010624 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464332998996991, + "loss": 3.1129, + "theoretical_loss": 4.205791774631566, + "tokens_seen": 266076160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046432296890672013, + "loss": 3.1271, + "theoretical_loss": 4.205667870274686, + "tokens_seen": 266141696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046431293881644937, + "loss": 3.0881, + "theoretical_loss": 4.2055440049654775, + "tokens_seen": 266207232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046430290872617855, + "loss": 3.1299, + "theoretical_loss": 4.205420178682024, + "tokens_seen": 266272768 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046429287863590773, + "loss": 2.8421, + "theoretical_loss": 4.205296391402431, + "tokens_seen": 266338304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642828485456369, + "loss": 2.9059, + "theoretical_loss": 4.205172643104818, + "tokens_seen": 266403840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046427281845536615, + "loss": 2.8721, + "theoretical_loss": 4.205048933767326, + "tokens_seen": 266469376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642627883650953, + "loss": 3.1183, + "theoretical_loss": 4.20492526336811, + "tokens_seen": 266534912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642527582748245, + "loss": 2.9405, + "theoretical_loss": 4.204801631885346, + "tokens_seen": 266600448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046424272818455364, + "loss": 2.9166, + "theoretical_loss": 4.204678039297223, + "tokens_seen": 266665984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046423269809428287, + "loss": 3.1439, + "theoretical_loss": 4.204554485581953, + "tokens_seen": 266731520 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046422266800401205, + "loss": 3.0142, + "theoretical_loss": 4.2044309707177625, + "tokens_seen": 266797056 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046421263791374123, + "loss": 3.166, + "theoretical_loss": 4.204307494682895, + "tokens_seen": 266862592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642026078234704, + "loss": 2.91, + "theoretical_loss": 4.204184057455613, + "tokens_seen": 266928128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641925777331996, + "loss": 2.7895, + "theoretical_loss": 4.204060659014197, + "tokens_seen": 266993664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 158889, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5656661987304688, + "objective/train/theoretical_loss": 4.2039372993369435, + "objective/train/tokens_used": 287519200, + "theoretical_loss": 4.2039372993369435, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641825476429288, + "loss": 2.9939, + "theoretical_loss": 4.2039372993369435, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464172517552658, + "loss": 2.7803, + "theoretical_loss": 4.203813978402167, + "tokens_seen": 267124736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046416248746238714, + "loss": 3.129, + "theoretical_loss": 4.203690696188198, + "tokens_seen": 267190272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641524573721164, + "loss": 3.0649, + "theoretical_loss": 4.203567452673388, + "tokens_seen": 267255808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641424272818455, + "loss": 3.0718, + "theoretical_loss": 4.203444247836103, + "tokens_seen": 267321344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046413239719157474, + "loss": 3.2325, + "theoretical_loss": 4.203321081654726, + "tokens_seen": 267386880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641223671013039, + "loss": 3.0568, + "theoretical_loss": 4.203197954107661, + "tokens_seen": 267452416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641123370110331, + "loss": 2.8217, + "theoretical_loss": 4.203074865173325, + "tokens_seen": 267517952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641023069207623, + "loss": 2.9454, + "theoretical_loss": 4.202951814830155, + "tokens_seen": 267583488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640922768304915, + "loss": 2.9256, + "theoretical_loss": 4.202828803056605, + "tokens_seen": 267649024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046408224674022064, + "loss": 2.9524, + "theoretical_loss": 4.202705829831144, + "tokens_seen": 267714560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640722166499499, + "loss": 2.9487, + "theoretical_loss": 4.202582895132262, + "tokens_seen": 267780096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464062186559679, + "loss": 2.6154, + "theoretical_loss": 4.202459998938464, + "tokens_seen": 267845632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046405215646940824, + "loss": 3.114, + "theoretical_loss": 4.202337141228272, + "tokens_seen": 267911168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640421263791374, + "loss": 3.0631, + "theoretical_loss": 4.202214321980227, + "tokens_seen": 267976704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640320962888666, + "loss": 2.9271, + "theoretical_loss": 4.202091541172885, + "tokens_seen": 268042240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640220661985958, + "loss": 2.9358, + "theoretical_loss": 4.201968798784821, + "tokens_seen": 268107776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046401203610832496, + "loss": 2.9304, + "theoretical_loss": 4.201846094794627, + "tokens_seen": 268173312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046400200601805414, + "loss": 3.2132, + "theoretical_loss": 4.201723429180911, + "tokens_seen": 268238848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639919759277834, + "loss": 2.8554, + "theoretical_loss": 4.2016008019222975, + "tokens_seen": 268304384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639819458375125, + "loss": 3.0379, + "theoretical_loss": 4.201478212997431, + "tokens_seen": 268369920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046397191574724174, + "loss": 2.8709, + "theoretical_loss": 4.201355662384971, + "tokens_seen": 268435456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639618856569709, + "loss": 3.0275, + "theoretical_loss": 4.2012331500635955, + "tokens_seen": 268500992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639518555667001, + "loss": 3.1757, + "theoretical_loss": 4.201110676011998, + "tokens_seen": 268566528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046394182547642934, + "loss": 3.3162, + "theoretical_loss": 4.200988240208889, + "tokens_seen": 268632064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 159274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.012082099914551, + "objective/train/theoretical_loss": 4.2008658426329974, + "objective/train/tokens_used": 289157600, + "theoretical_loss": 4.2008658426329974, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393179538615847, + "loss": 2.954, + "theoretical_loss": 4.2008658426329974, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639217652958877, + "loss": 2.9394, + "theoretical_loss": 4.200743483263068, + "tokens_seen": 268763136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639117352056169, + "loss": 2.9931, + "theoretical_loss": 4.200621162077864, + "tokens_seen": 268828672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046390170511534606, + "loss": 3.1958, + "theoretical_loss": 4.200498879056164, + "tokens_seen": 268894208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389167502507525, + "loss": 3.0091, + "theoretical_loss": 4.200376634176765, + "tokens_seen": 268959744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638816449348044, + "loss": 2.9129, + "theoretical_loss": 4.200254427418478, + "tokens_seen": 269025280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638716148445336, + "loss": 3.0744, + "theoretical_loss": 4.200132258760135, + "tokens_seen": 269090816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046386158475426284, + "loss": 3.0598, + "theoretical_loss": 4.200010128180582, + "tokens_seen": 269156352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385155466399197, + "loss": 3.3544, + "theoretical_loss": 4.1998880356586845, + "tokens_seen": 269221888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638415245737212, + "loss": 3.2152, + "theoretical_loss": 4.1997659811733214, + "tokens_seen": 269287424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046383149448345033, + "loss": 3.3844, + "theoretical_loss": 4.199643964703391, + "tokens_seen": 269352960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046382146439317957, + "loss": 3.2429, + "theoretical_loss": 4.199521986227808, + "tokens_seen": 269418496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381143430290875, + "loss": 3.002, + "theoretical_loss": 4.199400045725502, + "tokens_seen": 269484032 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046380140421263793, + "loss": 2.9083, + "theoretical_loss": 4.199278143175423, + "tokens_seen": 269549568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637913741223671, + "loss": 3.2151, + "theoretical_loss": 4.1991562785565355, + "tokens_seen": 269615104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046378134403209635, + "loss": 2.7854, + "theoretical_loss": 4.199034451847821, + "tokens_seen": 269680640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637713139418255, + "loss": 3.1079, + "theoretical_loss": 4.198912663028278, + "tokens_seen": 269746176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637612838515547, + "loss": 3.2046, + "theoretical_loss": 4.19879091207692, + "tokens_seen": 269811712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375125376128384, + "loss": 2.8742, + "theoretical_loss": 4.19866919897278, + "tokens_seen": 269877248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046374122367101307, + "loss": 2.8417, + "theoretical_loss": 4.1985475236949075, + "tokens_seen": 269942784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046373119358074225, + "loss": 3.0719, + "theoretical_loss": 4.198425886222367, + "tokens_seen": 270008320 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046372116349047143, + "loss": 3.1394, + "theoretical_loss": 4.198304286534239, + "tokens_seen": 270073856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637111334002006, + "loss": 2.8706, + "theoretical_loss": 4.198182724609624, + "tokens_seen": 270139392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637011033099298, + "loss": 2.9891, + "theoretical_loss": 4.198061200427635, + "tokens_seen": 270204928 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463691073219659, + "loss": 2.9082, + "theoretical_loss": 4.197939713967407, + "tokens_seen": 270270464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 160520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.315969467163086, + "objective/train/theoretical_loss": 4.197818265208085, + "objective/train/tokens_used": 290796000, + "theoretical_loss": 4.197818265208085, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636810431293882, + "loss": 2.939, + "theoretical_loss": 4.197818265208085, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046367101303911734, + "loss": 2.9865, + "theoretical_loss": 4.197696854128836, + "tokens_seen": 270401536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636609829488466, + "loss": 2.9617, + "theoretical_loss": 4.1975754807088395, + "tokens_seen": 270467072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636509528585757, + "loss": 3.1313, + "theoretical_loss": 4.1974541449272955, + "tokens_seen": 270532608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046364092276830494, + "loss": 2.8589, + "theoretical_loss": 4.197332846763418, + "tokens_seen": 270598144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636308926780341, + "loss": 3.168, + "theoretical_loss": 4.197211586196437, + "tokens_seen": 270663680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636208625877633, + "loss": 3.1609, + "theoretical_loss": 4.197090363205602, + "tokens_seen": 270729216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636108324974925, + "loss": 2.8238, + "theoretical_loss": 4.196969177770175, + "tokens_seen": 270794752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636008024072217, + "loss": 3.0672, + "theoretical_loss": 4.196848029869437, + "tokens_seen": 270860288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359077231695084, + "loss": 3.1223, + "theoretical_loss": 4.1967269194826855, + "tokens_seen": 270925824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635807422266801, + "loss": 3.1919, + "theoretical_loss": 4.196605846589234, + "tokens_seen": 270991360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635707121364092, + "loss": 3.1143, + "theoretical_loss": 4.196484811168412, + "tokens_seen": 271056896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046356068204613844, + "loss": 2.7825, + "theoretical_loss": 4.196363813199565, + "tokens_seen": 271122432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635506519558676, + "loss": 3.1101, + "theoretical_loss": 4.196242852662056, + "tokens_seen": 271187968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635406218655968, + "loss": 3.1081, + "theoretical_loss": 4.1961219295352645, + "tokens_seen": 271253504 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463530591775326, + "loss": 3.1529, + "theoretical_loss": 4.196001043798585, + "tokens_seen": 271319040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046352056168505516, + "loss": 3.1475, + "theoretical_loss": 4.195880195431428, + "tokens_seen": 271384576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046351053159478434, + "loss": 2.9622, + "theoretical_loss": 4.195759384413224, + "tokens_seen": 271450112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635005015045136, + "loss": 2.9907, + "theoretical_loss": 4.195638610723415, + "tokens_seen": 271515648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634904714142427, + "loss": 2.7124, + "theoretical_loss": 4.195517874341462, + "tokens_seen": 271581184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046348044132397194, + "loss": 2.9731, + "theoretical_loss": 4.195397175246841, + "tokens_seen": 271646720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046347041123370107, + "loss": 3.1014, + "theoretical_loss": 4.195276513419046, + "tokens_seen": 271712256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634603811434303, + "loss": 2.8613, + "theoretical_loss": 4.195155888837585, + "tokens_seen": 271777792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634503510531595, + "loss": 3.0449, + "theoretical_loss": 4.195035301481984, + "tokens_seen": 271843328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046344032096288867, + "loss": 2.7986, + "theoretical_loss": 4.194914751331785, + "tokens_seen": 271908864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 160942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2946550846099854, + "objective/train/theoretical_loss": 4.194794238366544, + "objective/train/tokens_used": 292434400, + "theoretical_loss": 4.194794238366544, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343029087261785, + "loss": 3.0722, + "theoretical_loss": 4.194794238366544, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634202607823471, + "loss": 3.0775, + "theoretical_loss": 4.194673762565837, + "tokens_seen": 272039936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634102306920762, + "loss": 2.9018, + "theoretical_loss": 4.194553323909252, + "tokens_seen": 272105472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046340020060180545, + "loss": 3.1252, + "theoretical_loss": 4.1944329223763965, + "tokens_seen": 272171008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046339017051153457, + "loss": 2.9432, + "theoretical_loss": 4.194312557946892, + "tokens_seen": 272236544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633801404212638, + "loss": 3.1044, + "theoretical_loss": 4.194192230600376, + "tokens_seen": 272302080 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463370110330993, + "loss": 3.0371, + "theoretical_loss": 4.194071940316505, + "tokens_seen": 272367616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046336008024072217, + "loss": 3.1066, + "theoretical_loss": 4.193951687074948, + "tokens_seen": 272433152 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046335005015045135, + "loss": 2.9569, + "theoretical_loss": 4.193831470855391, + "tokens_seen": 272498688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046334002006018053, + "loss": 3.1706, + "theoretical_loss": 4.193711291637538, + "tokens_seen": 272564224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633299899699097, + "loss": 3.0875, + "theoretical_loss": 4.1935911494011044, + "tokens_seen": 272629760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046331995987963895, + "loss": 3.0274, + "theoretical_loss": 4.193471044125828, + "tokens_seen": 272695296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633099297893681, + "loss": 3.1833, + "theoretical_loss": 4.193350975791458, + "tokens_seen": 272760832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632998996990973, + "loss": 3.0265, + "theoretical_loss": 4.193230944377762, + "tokens_seen": 272826368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046328986960882644, + "loss": 2.999, + "theoretical_loss": 4.1931109498645185, + "tokens_seen": 272891904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632798395185557, + "loss": 2.8691, + "theoretical_loss": 4.19299099223153, + "tokens_seen": 272957440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046326980942828485, + "loss": 3.1468, + "theoretical_loss": 4.192871071458608, + "tokens_seen": 273022976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046325977933801404, + "loss": 3.101, + "theoretical_loss": 4.192751187525585, + "tokens_seen": 273088512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632497492477432, + "loss": 2.8904, + "theoretical_loss": 4.192631340412304, + "tokens_seen": 273154048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046323971915747245, + "loss": 2.8277, + "theoretical_loss": 4.192511530098629, + "tokens_seen": 273219584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632296890672016, + "loss": 3.0174, + "theoretical_loss": 4.192391756564437, + "tokens_seen": 273285120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632196589769308, + "loss": 2.9337, + "theoretical_loss": 4.19227201978962, + "tokens_seen": 273350656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046320962888666, + "loss": 3.2491, + "theoretical_loss": 4.19215231975409, + "tokens_seen": 273416192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631995987963892, + "loss": 3.1248, + "theoretical_loss": 4.192032656437769, + "tokens_seen": 273481728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631895687061184, + "loss": 3.1044, + "theoretical_loss": 4.191913029820601, + "tokens_seen": 273547264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 162274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1903703212738037, + "objective/train/theoretical_loss": 4.19179343988254, + "objective/train/tokens_used": 294072800, + "theoretical_loss": 4.19179343988254, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046317953861584754, + "loss": 3.1483, + "theoretical_loss": 4.19179343988254, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631695085255768, + "loss": 2.8653, + "theoretical_loss": 4.191673886603559, + "tokens_seen": 273678336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631594784353059, + "loss": 3.0805, + "theoretical_loss": 4.191554369963647, + "tokens_seen": 273743872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046314944834503514, + "loss": 2.9344, + "theoretical_loss": 4.191434889942807, + "tokens_seen": 273809408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631394182547643, + "loss": 3.0085, + "theoretical_loss": 4.191315446521058, + "tokens_seen": 273874944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631293881644935, + "loss": 2.9289, + "theoretical_loss": 4.191196039678436, + "tokens_seen": 273940480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631193580742227, + "loss": 3.1368, + "theoretical_loss": 4.191076669394992, + "tokens_seen": 274006016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631093279839519, + "loss": 3.1613, + "theoretical_loss": 4.190957335650791, + "tokens_seen": 274071552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046309929789368104, + "loss": 2.8155, + "theoretical_loss": 4.190838038425916, + "tokens_seen": 274137088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630892678034103, + "loss": 3.1902, + "theoretical_loss": 4.190718777700465, + "tokens_seen": 274202624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630792377131394, + "loss": 3.0513, + "theoretical_loss": 4.190599553454551, + "tokens_seen": 274268160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046306920762286864, + "loss": 3.0037, + "theoretical_loss": 4.190480365668304, + "tokens_seen": 274333696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630591775325978, + "loss": 2.8063, + "theoretical_loss": 4.190361214321866, + "tokens_seen": 274399232 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463049147442327, + "loss": 2.9742, + "theoretical_loss": 4.190242099395399, + "tokens_seen": 274464768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630391173520562, + "loss": 2.9558, + "theoretical_loss": 4.190123020869077, + "tokens_seen": 274530304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046302908726178536, + "loss": 2.9187, + "theoretical_loss": 4.190003978723093, + "tokens_seen": 274595840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046301905717151455, + "loss": 3.0974, + "theoretical_loss": 4.189884972937652, + "tokens_seen": 274661376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630090270812438, + "loss": 3.1408, + "theoretical_loss": 4.189766003492977, + "tokens_seen": 274726912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629989969909729, + "loss": 3.0551, + "theoretical_loss": 4.189647070369304, + "tokens_seen": 274792448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298896690070214, + "loss": 3.1843, + "theoretical_loss": 4.189528173546888, + "tokens_seen": 274857984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046297893681043127, + "loss": 3.0686, + "theoretical_loss": 4.189409313005997, + "tokens_seen": 274923520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629689067201605, + "loss": 3.0667, + "theoretical_loss": 4.189290488726913, + "tokens_seen": 274989056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629588766298897, + "loss": 3.2895, + "theoretical_loss": 4.189171700689937, + "tokens_seen": 275054592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294884653961887, + "loss": 3.0592, + "theoretical_loss": 4.189052948875384, + "tokens_seen": 275120128 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046293881644934805, + "loss": 2.9131, + "theoretical_loss": 4.188934233263582, + "tokens_seen": 275185664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 163440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.128859519958496, + "objective/train/theoretical_loss": 4.188815553834879, + "objective/train/tokens_used": 295711200, + "theoretical_loss": 4.188815553834879, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629287863590773, + "loss": 3.0631, + "theoretical_loss": 4.188815553834879, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629187562688064, + "loss": 2.8185, + "theoretical_loss": 4.188696910569634, + "tokens_seen": 275316736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046290872617853565, + "loss": 3.0597, + "theoretical_loss": 4.188578303448223, + "tokens_seen": 275382272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046289869608826477, + "loss": 3.0644, + "theoretical_loss": 4.188459732451038, + "tokens_seen": 275447808 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462888665997994, + "loss": 3.2541, + "theoretical_loss": 4.188341197558486, + "tokens_seen": 275513344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628786359077232, + "loss": 2.772, + "theoretical_loss": 4.188222698750988, + "tokens_seen": 275578880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046286860581745237, + "loss": 2.945, + "theoretical_loss": 4.188104236008982, + "tokens_seen": 275644416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046285857572718155, + "loss": 2.9857, + "theoretical_loss": 4.18798580931292, + "tokens_seen": 275709952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284854563691073, + "loss": 2.8546, + "theoretical_loss": 4.18786741864327, + "tokens_seen": 275775488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628385155466399, + "loss": 3.1056, + "theoretical_loss": 4.187749063980516, + "tokens_seen": 275841024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046282848545636915, + "loss": 2.8899, + "theoretical_loss": 4.187630745305154, + "tokens_seen": 275906560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628184553660983, + "loss": 2.9852, + "theoretical_loss": 4.187512462597699, + "tokens_seen": 275972096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628084252758275, + "loss": 3.1059, + "theoretical_loss": 4.187394215838678, + "tokens_seen": 276037632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046279839518555664, + "loss": 2.945, + "theoretical_loss": 4.187276005008637, + "tokens_seen": 276103168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627883650952859, + "loss": 3.1584, + "theoretical_loss": 4.1871578300881325, + "tokens_seen": 276168704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046277833500501505, + "loss": 3.1305, + "theoretical_loss": 4.18703969105774, + "tokens_seen": 276234240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046276830491474424, + "loss": 2.8816, + "theoretical_loss": 4.186921587898048, + "tokens_seen": 276299776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627582748244734, + "loss": 2.988, + "theoretical_loss": 4.1868035205896605, + "tokens_seen": 276365312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274824473420265, + "loss": 2.8155, + "theoretical_loss": 4.186685489113197, + "tokens_seen": 276430848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627382146439318, + "loss": 3.1242, + "theoretical_loss": 4.186567493449292, + "tokens_seen": 276496384 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462728184553661, + "loss": 3.0243, + "theoretical_loss": 4.186449533578595, + "tokens_seen": 276561920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046271815446339014, + "loss": 2.9596, + "theoretical_loss": 4.186331609481769, + "tokens_seen": 276627456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627081243731194, + "loss": 3.3328, + "theoretical_loss": 4.186213721139496, + "tokens_seen": 276692992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046269809428284856, + "loss": 3.114, + "theoretical_loss": 4.186095868532469, + "tokens_seen": 276758528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046268806419257774, + "loss": 2.8937, + "theoretical_loss": 4.1859780516413965, + "tokens_seen": 276824064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 163970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.389463186264038, + "objective/train/theoretical_loss": 4.185860270447004, + "objective/train/tokens_used": 297349600, + "theoretical_loss": 4.185860270447004, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626780341023069, + "loss": 3.0341, + "theoretical_loss": 4.185860270447004, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626680040120361, + "loss": 3.1028, + "theoretical_loss": 4.185742524930031, + "tokens_seen": 276955136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626579739217653, + "loss": 2.9541, + "theoretical_loss": 4.185624815071232, + "tokens_seen": 277020672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626479438314945, + "loss": 2.9788, + "theoretical_loss": 4.185507140851375, + "tokens_seen": 277086208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046263791374122364, + "loss": 3.0129, + "theoretical_loss": 4.185389502251246, + "tokens_seen": 277151744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626278836509529, + "loss": 2.8961, + "theoretical_loss": 4.185271899251642, + "tokens_seen": 277217280 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462617853560682, + "loss": 2.8754, + "theoretical_loss": 4.185154331833379, + "tokens_seen": 277282816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260782347041124, + "loss": 2.9643, + "theoretical_loss": 4.185036799977285, + "tokens_seen": 277348352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625977933801404, + "loss": 3.0866, + "theoretical_loss": 4.184919303664203, + "tokens_seen": 277413888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625877632898696, + "loss": 3.1173, + "theoretical_loss": 4.184801842874992, + "tokens_seen": 277479424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625777331995988, + "loss": 3.0844, + "theoretical_loss": 4.1846844175905265, + "tokens_seen": 277544960 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462567703109328, + "loss": 3.0707, + "theoretical_loss": 4.184567027791693, + "tokens_seen": 277610496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046255767301905715, + "loss": 2.9221, + "theoretical_loss": 4.184449673459395, + "tokens_seen": 277676032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625476429287864, + "loss": 3.0805, + "theoretical_loss": 4.1843323545745505, + "tokens_seen": 277741568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625376128385155, + "loss": 2.9624, + "theoretical_loss": 4.184215071118093, + "tokens_seen": 277807104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252758274824475, + "loss": 2.9347, + "theoretical_loss": 4.184097823070967, + "tokens_seen": 277872640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625175526579739, + "loss": 3.0535, + "theoretical_loss": 4.183980610414137, + "tokens_seen": 277938176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625075225677031, + "loss": 2.9498, + "theoretical_loss": 4.18386343312858, + "tokens_seen": 278003712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624974924774323, + "loss": 2.9249, + "theoretical_loss": 4.183746291195285, + "tokens_seen": 278069248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046248746238716147, + "loss": 3.0889, + "theoretical_loss": 4.18362918459526, + "tokens_seen": 278134784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046247743229689065, + "loss": 2.9025, + "theoretical_loss": 4.183512113309525, + "tokens_seen": 278200320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624674022066199, + "loss": 3.1737, + "theoretical_loss": 4.1833950773191155, + "tokens_seen": 278265856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046245737211634907, + "loss": 2.8199, + "theoretical_loss": 4.183278076605083, + "tokens_seen": 278331392 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244734202607825, + "loss": 3.1762, + "theoretical_loss": 4.18316111114849, + "tokens_seen": 278396928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624373119358075, + "loss": 3.1915, + "theoretical_loss": 4.183044180930418, + "tokens_seen": 278462464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 165182, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.31074857711792, + "objective/train/theoretical_loss": 4.182927285931959, + "objective/train/tokens_used": 298988000, + "theoretical_loss": 4.182927285931959, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624272818455366, + "loss": 3.0716, + "theoretical_loss": 4.182927285931959, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046241725175526585, + "loss": 2.8782, + "theoretical_loss": 4.182810426134223, + "tokens_seen": 278593536 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462407221664995, + "loss": 3.1372, + "theoretical_loss": 4.1826936015183325, + "tokens_seen": 278659072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623971915747242, + "loss": 3.0399, + "theoretical_loss": 4.182576812065426, + "tokens_seen": 278724608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623871614844534, + "loss": 3.1069, + "theoretical_loss": 4.1824600577566535, + "tokens_seen": 278790144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046237713139418257, + "loss": 3.003, + "theoretical_loss": 4.182343338573185, + "tokens_seen": 278855680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236710130391175, + "loss": 2.981, + "theoretical_loss": 4.1822266544961995, + "tokens_seen": 278921216 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046235707121364093, + "loss": 2.9963, + "theoretical_loss": 4.1821100055068925, + "tokens_seen": 278986752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623470411233701, + "loss": 2.8282, + "theoretical_loss": 4.181993391586476, + "tokens_seen": 279052288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046233701103309935, + "loss": 2.9174, + "theoretical_loss": 4.181876812716174, + "tokens_seen": 279117824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623269809428285, + "loss": 3.0528, + "theoretical_loss": 4.181760268877226, + "tokens_seen": 279183360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623169508525577, + "loss": 3.0571, + "theoretical_loss": 4.181643760050885, + "tokens_seen": 279248896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230692076228684, + "loss": 3.0749, + "theoretical_loss": 4.181527286218419, + "tokens_seen": 279314432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622968906720161, + "loss": 3.0104, + "theoretical_loss": 4.181410847361111, + "tokens_seen": 279379968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046228686058174525, + "loss": 3.1389, + "theoretical_loss": 4.181294443460257, + "tokens_seen": 279445504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046227683049147444, + "loss": 2.8566, + "theoretical_loss": 4.18117807449717, + "tokens_seen": 279511040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622668004012036, + "loss": 3.1739, + "theoretical_loss": 4.181061740453174, + "tokens_seen": 279576576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046225677031093285, + "loss": 3.1752, + "theoretical_loss": 4.18094544130961, + "tokens_seen": 279642112 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462246740220662, + "loss": 2.9331, + "theoretical_loss": 4.180829177047832, + "tokens_seen": 279707648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622367101303912, + "loss": 2.8764, + "theoretical_loss": 4.180712947649208, + "tokens_seen": 279773184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046222668004012034, + "loss": 3.1994, + "theoretical_loss": 4.180596753095122, + "tokens_seen": 279838720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622166499498496, + "loss": 3.2665, + "theoretical_loss": 4.18048059336697, + "tokens_seen": 279904256 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046220661985957876, + "loss": 2.9311, + "theoretical_loss": 4.180364468446165, + "tokens_seen": 279969792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046219658976930794, + "loss": 3.0726, + "theoretical_loss": 4.180248378314133, + "tokens_seen": 280035328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621865596790371, + "loss": 3.0656, + "theoretical_loss": 4.180132322952312, + "tokens_seen": 280100864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 165816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.640207529067993, + "objective/train/theoretical_loss": 4.180016302342159, + "objective/train/tokens_used": 300626400, + "theoretical_loss": 4.180016302342159, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621765295887663, + "loss": 2.8359, + "theoretical_loss": 4.180016302342159, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621664994984955, + "loss": 3.0966, + "theoretical_loss": 4.179900316465139, + "tokens_seen": 280231936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621564694082247, + "loss": 3.0808, + "theoretical_loss": 4.179784365302739, + "tokens_seen": 280297472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214643931795384, + "loss": 2.9101, + "theoretical_loss": 4.179668448836454, + "tokens_seen": 280363008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621364092276831, + "loss": 3.1295, + "theoretical_loss": 4.179552567047794, + "tokens_seen": 280428544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621263791374122, + "loss": 2.9127, + "theoretical_loss": 4.179436719918286, + "tokens_seen": 280494080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046211634904714144, + "loss": 3.2209, + "theoretical_loss": 4.179320907429468, + "tokens_seen": 280559616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621063189568706, + "loss": 3.0106, + "theoretical_loss": 4.179205129562895, + "tokens_seen": 280625152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620962888665998, + "loss": 3.0269, + "theoretical_loss": 4.179089386300134, + "tokens_seen": 280690688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462086258776329, + "loss": 3.1179, + "theoretical_loss": 4.178973677622768, + "tokens_seen": 280756224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620762286860582, + "loss": 3.278, + "theoretical_loss": 4.17885800351239, + "tokens_seen": 280821760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046206619859578735, + "loss": 2.9086, + "theoretical_loss": 4.178742363950614, + "tokens_seen": 280887296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620561685055166, + "loss": 3.1124, + "theoretical_loss": 4.178626758919062, + "tokens_seen": 280952832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620461384152457, + "loss": 3.0546, + "theoretical_loss": 4.178511188399372, + "tokens_seen": 281018368 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046203610832497495, + "loss": 3.1334, + "theoretical_loss": 4.1783956523731955, + "tokens_seen": 281083904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620260782347041, + "loss": 2.9334, + "theoretical_loss": 4.178280150822202, + "tokens_seen": 281149440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620160481444333, + "loss": 2.8178, + "theoretical_loss": 4.178164683728068, + "tokens_seen": 281214976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620060180541625, + "loss": 2.8805, + "theoretical_loss": 4.17804925107249, + "tokens_seen": 281280512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046199598796389167, + "loss": 2.919, + "theoretical_loss": 4.1779338528371746, + "tokens_seen": 281346048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046198595787362085, + "loss": 3.1013, + "theoretical_loss": 4.177818489003846, + "tokens_seen": 281411584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619759277833501, + "loss": 3.1344, + "theoretical_loss": 4.177703159554238, + "tokens_seen": 281477120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619658976930792, + "loss": 3.0054, + "theoretical_loss": 4.177587864470103, + "tokens_seen": 281542656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046195586760280845, + "loss": 2.8447, + "theoretical_loss": 4.1774726037332055, + "tokens_seen": 281608192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046194583751253763, + "loss": 3.0365, + "theoretical_loss": 4.17735737732532, + "tokens_seen": 281673728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619358074222668, + "loss": 3.0124, + "theoretical_loss": 4.177242185228242, + "tokens_seen": 281739264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 166963, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.019186019897461, + "objective/train/theoretical_loss": 4.177127027423776, + "objective/train/tokens_used": 302264800, + "theoretical_loss": 4.177127027423776, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461925777331996, + "loss": 2.9856, + "theoretical_loss": 4.177127027423776, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619157472417252, + "loss": 3.0215, + "theoretical_loss": 4.17701190389374, + "tokens_seen": 281870336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046190571715145435, + "loss": 2.9971, + "theoretical_loss": 4.176896814619969, + "tokens_seen": 281935872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618956870611836, + "loss": 3.0423, + "theoretical_loss": 4.176781759584312, + "tokens_seen": 282001408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618856569709127, + "loss": 3.0798, + "theoretical_loss": 4.176666738768627, + "tokens_seen": 282066944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046187562688064195, + "loss": 2.8872, + "theoretical_loss": 4.176551752154791, + "tokens_seen": 282132480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618655967903711, + "loss": 3.1228, + "theoretical_loss": 4.1764367997246925, + "tokens_seen": 282198016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618555667001003, + "loss": 3.0256, + "theoretical_loss": 4.1763218814602325, + "tokens_seen": 282263552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618455366098295, + "loss": 3.1382, + "theoretical_loss": 4.17620699734333, + "tokens_seen": 282329088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618355065195587, + "loss": 2.807, + "theoretical_loss": 4.176092147355915, + "tokens_seen": 282394624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046182547642928786, + "loss": 2.9633, + "theoretical_loss": 4.175977331479929, + "tokens_seen": 282460160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046181544633901704, + "loss": 3.0836, + "theoretical_loss": 4.175862549697332, + "tokens_seen": 282525696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618054162487462, + "loss": 3.0762, + "theoretical_loss": 4.175747801990093, + "tokens_seen": 282591232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046179538615847545, + "loss": 3.1594, + "theoretical_loss": 4.1756330883402, + "tokens_seen": 282656768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617853560682046, + "loss": 3.136, + "theoretical_loss": 4.1755184087296495, + "tokens_seen": 282722304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617753259779338, + "loss": 3.0347, + "theoretical_loss": 4.175403763140456, + "tokens_seen": 282787840 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461765295887663, + "loss": 2.9652, + "theoretical_loss": 4.175289151554643, + "tokens_seen": 282853376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617552657973922, + "loss": 3.0566, + "theoretical_loss": 4.175174573954252, + "tokens_seen": 282918912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046174523570712136, + "loss": 2.8965, + "theoretical_loss": 4.175060030321337, + "tokens_seen": 282984448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046173520561685054, + "loss": 3.0043, + "theoretical_loss": 4.174945520637964, + "tokens_seen": 283049984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617251755265797, + "loss": 3.124, + "theoretical_loss": 4.174831044886213, + "tokens_seen": 283115520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046171514543630896, + "loss": 3.0059, + "theoretical_loss": 4.17471660304818, + "tokens_seen": 283181056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046170511534603814, + "loss": 3.1171, + "theoretical_loss": 4.174602195105972, + "tokens_seen": 283246592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616950852557673, + "loss": 2.735, + "theoretical_loss": 4.174487821041711, + "tokens_seen": 283312128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616850551654965, + "loss": 2.9887, + "theoretical_loss": 4.174373480837532, + "tokens_seen": 283377664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 167624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3214924335479736, + "objective/train/theoretical_loss": 4.174259174475583, + "objective/train/tokens_used": 303903200, + "theoretical_loss": 4.174259174475583, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616750250752257, + "loss": 3.0628, + "theoretical_loss": 4.174259174475583, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616649949849549, + "loss": 3.1753, + "theoretical_loss": 4.174144901938025, + "tokens_seen": 283508736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046165496489468404, + "loss": 2.9032, + "theoretical_loss": 4.174030663207036, + "tokens_seen": 283574272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616449348044133, + "loss": 3.0913, + "theoretical_loss": 4.173916458264804, + "tokens_seen": 283639808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616349047141424, + "loss": 3.0559, + "theoretical_loss": 4.173802287093531, + "tokens_seen": 283705344 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046162487462387164, + "loss": 2.8405, + "theoretical_loss": 4.173688149675435, + "tokens_seen": 283770880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616148445336008, + "loss": 3.1749, + "theoretical_loss": 4.173574045992744, + "tokens_seen": 283836416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046160481444333, + "loss": 2.9452, + "theoretical_loss": 4.173459976027702, + "tokens_seen": 283901952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615947843530592, + "loss": 3.1586, + "theoretical_loss": 4.173345939762564, + "tokens_seen": 283967488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615847542627884, + "loss": 2.9694, + "theoretical_loss": 4.173231937179601, + "tokens_seen": 284033024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046157472417251755, + "loss": 3.2162, + "theoretical_loss": 4.173117968261097, + "tokens_seen": 284098560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615646940822468, + "loss": 3.1405, + "theoretical_loss": 4.173004032989347, + "tokens_seen": 284164096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615546639919759, + "loss": 3.0749, + "theoretical_loss": 4.172890131346662, + "tokens_seen": 284229632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046154463390170515, + "loss": 3.1627, + "theoretical_loss": 4.172776263315367, + "tokens_seen": 284295168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615346038114343, + "loss": 3.1345, + "theoretical_loss": 4.172662428877796, + "tokens_seen": 284360704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615245737211635, + "loss": 2.9964, + "theoretical_loss": 4.172548628016302, + "tokens_seen": 284426240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615145436308927, + "loss": 3.0997, + "theoretical_loss": 4.172434860713246, + "tokens_seen": 284491776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046150451354062187, + "loss": 3.1053, + "theoretical_loss": 4.172321126951008, + "tokens_seen": 284557312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046149448345035105, + "loss": 3.108, + "theoretical_loss": 4.172207426711974, + "tokens_seen": 284622848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614844533600803, + "loss": 2.8824, + "theoretical_loss": 4.1720937599785515, + "tokens_seen": 284688384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614744232698094, + "loss": 3.2381, + "theoretical_loss": 4.171980126733155, + "tokens_seen": 284753920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046146439317953865, + "loss": 3.1941, + "theoretical_loss": 4.171866526958215, + "tokens_seen": 284819456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046145436308926783, + "loss": 2.7743, + "theoretical_loss": 4.171752960636176, + "tokens_seen": 284884992 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461444332998997, + "loss": 2.9867, + "theoretical_loss": 4.171639427749492, + "tokens_seen": 284950528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614343029087262, + "loss": 3.0296, + "theoretical_loss": 4.171525928280635, + "tokens_seen": 285016064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 168853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6973540782928467, + "objective/train/theoretical_loss": 4.171412462212087, + "objective/train/tokens_used": 305541600, + "theoretical_loss": 4.171412462212087, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614242728184554, + "loss": 2.8165, + "theoretical_loss": 4.171412462212087, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046141424272818455, + "loss": 3.1382, + "theoretical_loss": 4.171299029526344, + "tokens_seen": 285147136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614042126379138, + "loss": 3.1788, + "theoretical_loss": 4.171185630205917, + "tokens_seen": 285212672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613941825476429, + "loss": 2.9897, + "theoretical_loss": 4.171072264233327, + "tokens_seen": 285278208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046138415245737215, + "loss": 3.0584, + "theoretical_loss": 4.17095893159111, + "tokens_seen": 285343744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613741223671013, + "loss": 3.0978, + "theoretical_loss": 4.170845632261815, + "tokens_seen": 285409280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613640922768305, + "loss": 2.9229, + "theoretical_loss": 4.1707323662280045, + "tokens_seen": 285474816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613540621865597, + "loss": 3.1211, + "theoretical_loss": 4.170619133472253, + "tokens_seen": 285540352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613440320962889, + "loss": 2.9585, + "theoretical_loss": 4.170505933977149, + "tokens_seen": 285605888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046133400200601806, + "loss": 3.1654, + "theoretical_loss": 4.1703927677252945, + "tokens_seen": 285671424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046132397191574724, + "loss": 3.116, + "theoretical_loss": 4.170279634699304, + "tokens_seen": 285736960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613139418254764, + "loss": 3.2018, + "theoretical_loss": 4.170166534881803, + "tokens_seen": 285802496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046130391173520566, + "loss": 3.0549, + "theoretical_loss": 4.170053468255435, + "tokens_seen": 285868032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612938816449348, + "loss": 2.795, + "theoretical_loss": 4.169940434802851, + "tokens_seen": 285933568 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461283851554664, + "loss": 3.0738, + "theoretical_loss": 4.16982743450672, + "tokens_seen": 285999104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612738214643932, + "loss": 3.0826, + "theoretical_loss": 4.169714467349721, + "tokens_seen": 286064640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612637913741224, + "loss": 2.9928, + "theoretical_loss": 4.1696015333145455, + "tokens_seen": 286130176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046125376128385156, + "loss": 3.0505, + "theoretical_loss": 4.169488632383901, + "tokens_seen": 286195712 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046124373119358074, + "loss": 3.1082, + "theoretical_loss": 4.169375764540505, + "tokens_seen": 286261248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612337011033099, + "loss": 2.9721, + "theoretical_loss": 4.16926292976709, + "tokens_seen": 286326784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046122367101303916, + "loss": 3.1107, + "theoretical_loss": 4.1691501280464, + "tokens_seen": 286392320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612136409227683, + "loss": 3.0903, + "theoretical_loss": 4.1690373593611945, + "tokens_seen": 286457856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612036108324975, + "loss": 2.9085, + "theoretical_loss": 4.168924623694241, + "tokens_seen": 286523392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046119358074222665, + "loss": 2.9324, + "theoretical_loss": 4.168811921028325, + "tokens_seen": 286588928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611835506519559, + "loss": 2.9307, + "theoretical_loss": 4.168699251346243, + "tokens_seen": 286654464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 169449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8505656719207764, + "objective/train/theoretical_loss": 4.168586614630803, + "objective/train/tokens_used": 307180000, + "theoretical_loss": 4.168586614630803, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046117352056168506, + "loss": 3.006, + "theoretical_loss": 4.168586614630803, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046116349047141425, + "loss": 2.8379, + "theoretical_loss": 4.168474010864829, + "tokens_seen": 286785536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611534603811434, + "loss": 2.8133, + "theoretical_loss": 4.168361440031154, + "tokens_seen": 286851072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611434302908726, + "loss": 2.8832, + "theoretical_loss": 4.1682489021126266, + "tokens_seen": 286916608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611334002006018, + "loss": 3.0822, + "theoretical_loss": 4.168136397092109, + "tokens_seen": 286982144 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461123370110331, + "loss": 3.1401, + "theoretical_loss": 4.168023924952473, + "tokens_seen": 287047680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046111334002006015, + "loss": 3.0236, + "theoretical_loss": 4.167911485676606, + "tokens_seen": 287113216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611033099297894, + "loss": 2.9008, + "theoretical_loss": 4.1677990792474064, + "tokens_seen": 287178752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046109327983951857, + "loss": 3.0368, + "theoretical_loss": 4.167686705647787, + "tokens_seen": 287244288 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046108324974924775, + "loss": 2.8824, + "theoretical_loss": 4.167574364860673, + "tokens_seen": 287309824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046107321965897693, + "loss": 2.8937, + "theoretical_loss": 4.167462056869001, + "tokens_seen": 287375360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610631895687061, + "loss": 3.037, + "theoretical_loss": 4.167349781655721, + "tokens_seen": 287440896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610531594784353, + "loss": 3.0067, + "theoretical_loss": 4.167237539203798, + "tokens_seen": 287506432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610431293881645, + "loss": 2.8215, + "theoretical_loss": 4.167125329496207, + "tokens_seen": 287571968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046103309929789365, + "loss": 3.092, + "theoretical_loss": 4.167013152515935, + "tokens_seen": 287637504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610230692076229, + "loss": 2.834, + "theoretical_loss": 4.1669010082459845, + "tokens_seen": 287703040 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461013039117352, + "loss": 3.0378, + "theoretical_loss": 4.16678889666937, + "tokens_seen": 287768576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046100300902708125, + "loss": 2.9538, + "theoretical_loss": 4.166676817769118, + "tokens_seen": 287834112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046099297893681043, + "loss": 2.936, + "theoretical_loss": 4.166564771528268, + "tokens_seen": 287899648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609829488465396, + "loss": 3.0104, + "theoretical_loss": 4.16645275792987, + "tokens_seen": 287965184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609729187562688, + "loss": 3.0223, + "theoretical_loss": 4.166340776956991, + "tokens_seen": 288030720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046096288866599803, + "loss": 2.9447, + "theoretical_loss": 4.166228828592708, + "tokens_seen": 288096256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609528585757272, + "loss": 2.9425, + "theoretical_loss": 4.166116912820109, + "tokens_seen": 288161792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609428284854564, + "loss": 3.0139, + "theoretical_loss": 4.166005029622298, + "tokens_seen": 288227328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609327983951856, + "loss": 2.7581, + "theoretical_loss": 4.16589317898239, + "tokens_seen": 288292864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 170373, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9227163791656494, + "objective/train/theoretical_loss": 4.165781360883513, + "objective/train/tokens_used": 308818400, + "theoretical_loss": 4.165781360883513, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046092276830491475, + "loss": 3.0162, + "theoretical_loss": 4.165781360883513, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460912738214644, + "loss": 2.9972, + "theoretical_loss": 4.165669575308807, + "tokens_seen": 288423936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609027081243731, + "loss": 2.9231, + "theoretical_loss": 4.165557822241425, + "tokens_seen": 288489472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046089267803410235, + "loss": 2.9746, + "theoretical_loss": 4.165446101664531, + "tokens_seen": 288555008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608826479438315, + "loss": 3.0331, + "theoretical_loss": 4.165334413561306, + "tokens_seen": 288620544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608726178535607, + "loss": 3.0425, + "theoretical_loss": 4.165222757914936, + "tokens_seen": 288686080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608625877632899, + "loss": 2.9917, + "theoretical_loss": 4.165111134708629, + "tokens_seen": 288751616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608525576730191, + "loss": 3.0351, + "theoretical_loss": 4.164999543925597, + "tokens_seen": 288817152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084252758274826, + "loss": 2.9461, + "theoretical_loss": 4.164887985549068, + "tokens_seen": 288882688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046083249749247744, + "loss": 3.0879, + "theoretical_loss": 4.164776459562285, + "tokens_seen": 288948224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608224674022066, + "loss": 2.9828, + "theoretical_loss": 4.164664965948499, + "tokens_seen": 289013760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046081243731193586, + "loss": 3.1005, + "theoretical_loss": 4.164553504690976, + "tokens_seen": 289079296 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460802407221665, + "loss": 3.232, + "theoretical_loss": 4.164442075772993, + "tokens_seen": 289144832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607923771313942, + "loss": 3.0206, + "theoretical_loss": 4.164330679177841, + "tokens_seen": 289210368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607823470411234, + "loss": 2.9927, + "theoretical_loss": 4.164219314888823, + "tokens_seen": 289275904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607723169508526, + "loss": 2.8879, + "theoretical_loss": 4.164107982889252, + "tokens_seen": 289341440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046076228686058176, + "loss": 2.8926, + "theoretical_loss": 4.163996683162458, + "tokens_seen": 289406976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046075225677031094, + "loss": 2.955, + "theoretical_loss": 4.163885415691779, + "tokens_seen": 289472512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607422266800401, + "loss": 2.9791, + "theoretical_loss": 4.163774180460569, + "tokens_seen": 289538048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046073219658976936, + "loss": 2.9538, + "theoretical_loss": 4.163662977452192, + "tokens_seen": 289603584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607221664994985, + "loss": 2.9461, + "theoretical_loss": 4.163551806650023, + "tokens_seen": 289669120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607121364092277, + "loss": 2.8711, + "theoretical_loss": 4.163440668037453, + "tokens_seen": 289734656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070210631895685, + "loss": 3.0029, + "theoretical_loss": 4.163329561597885, + "tokens_seen": 289800192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606920762286861, + "loss": 2.892, + "theoretical_loss": 4.1632184873147295, + "tokens_seen": 289865728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046068204613841526, + "loss": 2.936, + "theoretical_loss": 4.1631074451714145, + "tokens_seen": 289931264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 171595, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1895124912261963, + "objective/train/theoretical_loss": 4.162996435151379, + "objective/train/tokens_used": 310456800, + "theoretical_loss": 4.162996435151379, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046067201604814445, + "loss": 3.0838, + "theoretical_loss": 4.162996435151379, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606619859578736, + "loss": 2.8576, + "theoretical_loss": 4.1628854572380725, + "tokens_seen": 290062336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606519558676028, + "loss": 2.9221, + "theoretical_loss": 4.162774511414959, + "tokens_seen": 290127872 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460641925777332, + "loss": 2.9766, + "theoretical_loss": 4.162663597665513, + "tokens_seen": 290193408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606318956870612, + "loss": 3.1019, + "theoretical_loss": 4.162552715973222, + "tokens_seen": 290258944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046062186559679035, + "loss": 3.0226, + "theoretical_loss": 4.162441866321586, + "tokens_seen": 290324480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606118355065196, + "loss": 3.1766, + "theoretical_loss": 4.162331048694118, + "tokens_seen": 290390016 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060180541624877, + "loss": 2.9799, + "theoretical_loss": 4.162220263074341, + "tokens_seen": 290455552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046059177532597795, + "loss": 3.0867, + "theoretical_loss": 4.1621095094457905, + "tokens_seen": 290521088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046058174523570713, + "loss": 3.0066, + "theoretical_loss": 4.161998787792016, + "tokens_seen": 290586624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605717151454363, + "loss": 3.0877, + "theoretical_loss": 4.161888098096579, + "tokens_seen": 290652160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605616850551655, + "loss": 2.8912, + "theoretical_loss": 4.161777440343051, + "tokens_seen": 290717696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605516549648947, + "loss": 3.0279, + "theoretical_loss": 4.161666814515017, + "tokens_seen": 290783232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054162487462385, + "loss": 2.9034, + "theoretical_loss": 4.161556220596075, + "tokens_seen": 290848768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605315947843531, + "loss": 2.9769, + "theoretical_loss": 4.161445658569833, + "tokens_seen": 290914304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605215646940822, + "loss": 3.0383, + "theoretical_loss": 4.161335128419914, + "tokens_seen": 290979840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046051153460381145, + "loss": 2.9355, + "theoretical_loss": 4.16122463012995, + "tokens_seen": 291045376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050150451354063, + "loss": 3.299, + "theoretical_loss": 4.161114163683587, + "tokens_seen": 291110912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604914744232698, + "loss": 2.8822, + "theoretical_loss": 4.161003729064483, + "tokens_seen": 291176448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460481444332999, + "loss": 3.0251, + "theoretical_loss": 4.160893326256307, + "tokens_seen": 291241984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046047141424272823, + "loss": 2.8968, + "theoretical_loss": 4.160782955242741, + "tokens_seen": 291307520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046138415245736, + "loss": 3.0828, + "theoretical_loss": 4.160672616007479, + "tokens_seen": 291373056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604513540621866, + "loss": 3.2492, + "theoretical_loss": 4.160562308534226, + "tokens_seen": 291438592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604413239719157, + "loss": 2.7536, + "theoretical_loss": 4.160452032806701, + "tokens_seen": 291504128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046043129388164495, + "loss": 3.1393, + "theoretical_loss": 4.160341788808633, + "tokens_seen": 291569664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 172214, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0195140838623047, + "objective/train/theoretical_loss": 4.160231576523763, + "objective/train/tokens_used": 312095200, + "theoretical_loss": 4.160231576523763, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042126379137414, + "loss": 2.9725, + "theoretical_loss": 4.160231576523763, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604112337011033, + "loss": 3.0591, + "theoretical_loss": 4.160121395935847, + "tokens_seen": 291700736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604012036108325, + "loss": 2.6618, + "theoretical_loss": 4.160011247028649, + "tokens_seen": 291766272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603911735205617, + "loss": 2.9955, + "theoretical_loss": 4.159901129785947, + "tokens_seen": 291831808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046038114343029086, + "loss": 2.9759, + "theoretical_loss": 4.159791044191532, + "tokens_seen": 291897344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603711133400201, + "loss": 2.9241, + "theoretical_loss": 4.1596809902292025, + "tokens_seen": 291962880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603610832497492, + "loss": 3.0356, + "theoretical_loss": 4.159570967882775, + "tokens_seen": 292028416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046035105315947846, + "loss": 2.8435, + "theoretical_loss": 4.1594609771360735, + "tokens_seen": 292093952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603410230692076, + "loss": 3.1677, + "theoretical_loss": 4.159351017972936, + "tokens_seen": 292159488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603309929789368, + "loss": 2.7557, + "theoretical_loss": 4.159241090377211, + "tokens_seen": 292225024 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460320962888666, + "loss": 2.997, + "theoretical_loss": 4.15913119433276, + "tokens_seen": 292290560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603109327983952, + "loss": 3.1299, + "theoretical_loss": 4.159021329823457, + "tokens_seen": 292356096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030090270812436, + "loss": 2.9866, + "theoretical_loss": 4.158911496833184, + "tokens_seen": 292421632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602908726178536, + "loss": 2.8174, + "theoretical_loss": 4.15880169534584, + "tokens_seen": 292487168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602808425275827, + "loss": 3.0642, + "theoretical_loss": 4.158691925345334, + "tokens_seen": 292552704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046027081243731196, + "loss": 2.9502, + "theoretical_loss": 4.158582186815584, + "tokens_seen": 292618240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602607823470411, + "loss": 2.957, + "theoretical_loss": 4.158472479740524, + "tokens_seen": 292683776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602507522567703, + "loss": 2.9731, + "theoretical_loss": 4.158362804104097, + "tokens_seen": 292749312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602407221664995, + "loss": 3.0535, + "theoretical_loss": 4.1582531598902595, + "tokens_seen": 292814848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602306920762287, + "loss": 2.985, + "theoretical_loss": 4.158143547082978, + "tokens_seen": 292880384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046022066198595787, + "loss": 2.9562, + "theoretical_loss": 4.158033965666233, + "tokens_seen": 292945920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046021063189568705, + "loss": 2.8592, + "theoretical_loss": 4.157924415624014, + "tokens_seen": 293011456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602006018054163, + "loss": 2.9666, + "theoretical_loss": 4.157814896940324, + "tokens_seen": 293076992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046019057171514546, + "loss": 3.2116, + "theoretical_loss": 4.157705409599179, + "tokens_seen": 293142528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046018054162487465, + "loss": 2.9664, + "theoretical_loss": 4.157595953584604, + "tokens_seen": 293208064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 173342, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8450427055358887, + "objective/train/theoretical_loss": 4.1574865288806375, + "objective/train/tokens_used": 313733600, + "theoretical_loss": 4.1574865288806375, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601705115346038, + "loss": 2.9126, + "theoretical_loss": 4.1574865288806375, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460160481444333, + "loss": 2.7492, + "theoretical_loss": 4.157377135471328, + "tokens_seen": 293339136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601504513540622, + "loss": 2.8848, + "theoretical_loss": 4.157267773340738, + "tokens_seen": 293404672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601404212637914, + "loss": 3.0064, + "theoretical_loss": 4.15715844247294, + "tokens_seen": 293470208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046013039117352055, + "loss": 2.9549, + "theoretical_loss": 4.157049142852019, + "tokens_seen": 293535744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601203610832498, + "loss": 2.9487, + "theoretical_loss": 4.15693987446207, + "tokens_seen": 293601280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046011033099297897, + "loss": 3.0182, + "theoretical_loss": 4.156830637287203, + "tokens_seen": 293666816 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046010030090270815, + "loss": 2.9087, + "theoretical_loss": 4.156721431311535, + "tokens_seen": 293732352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046009027081243733, + "loss": 3.1088, + "theoretical_loss": 4.1566122565192, + "tokens_seen": 293797888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600802407221665, + "loss": 3.0616, + "theoretical_loss": 4.156503112894339, + "tokens_seen": 293863424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600702106318957, + "loss": 2.9094, + "theoretical_loss": 4.156394000421107, + "tokens_seen": 293928960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046006018054162493, + "loss": 3.0927, + "theoretical_loss": 4.15628491908367, + "tokens_seen": 293994496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046005015045135405, + "loss": 3.1256, + "theoretical_loss": 4.156175868866205, + "tokens_seen": 294060032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600401203610833, + "loss": 3.0551, + "theoretical_loss": 4.156066849752902, + "tokens_seen": 294125568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600300902708124, + "loss": 3.0325, + "theoretical_loss": 4.15595786172796, + "tokens_seen": 294191104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046002006018054165, + "loss": 3.0107, + "theoretical_loss": 4.155848904775594, + "tokens_seen": 294256640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046001003009027083, + "loss": 2.9292, + "theoretical_loss": 4.155739978880025, + "tokens_seen": 294322176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 2.831, + "theoretical_loss": 4.155631084025489, + "tokens_seen": 294387712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599899699097292, + "loss": 3.1475, + "theoretical_loss": 4.155522220196234, + "tokens_seen": 294453248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045997993981945843, + "loss": 2.9871, + "theoretical_loss": 4.155413387376518, + "tokens_seen": 294518784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045996990972918756, + "loss": 3.1932, + "theoretical_loss": 4.155304585550609, + "tokens_seen": 294584320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599598796389168, + "loss": 2.9215, + "theoretical_loss": 4.15519581470279, + "tokens_seen": 294649856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599498495486459, + "loss": 2.9558, + "theoretical_loss": 4.155087074817353, + "tokens_seen": 294715392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993981945837515, + "loss": 3.2962, + "theoretical_loss": 4.154978365878603, + "tokens_seen": 294780928 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045992978936810434, + "loss": 3.1586, + "theoretical_loss": 4.154869687870854, + "tokens_seen": 294846464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 173989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6182363033294678, + "objective/train/theoretical_loss": 4.1547610407784346, + "objective/train/tokens_used": 315372000, + "theoretical_loss": 4.1547610407784346, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599197592778335, + "loss": 3.244, + "theoretical_loss": 4.1547610407784346, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599097291875627, + "loss": 3.042, + "theoretical_loss": 4.154652424585683, + "tokens_seen": 294977536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598996990972919, + "loss": 2.722, + "theoretical_loss": 4.154543839276948, + "tokens_seen": 295043072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045988966900702106, + "loss": 3.0362, + "theoretical_loss": 4.154435284836591, + "tokens_seen": 295108608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598796389167503, + "loss": 2.8414, + "theoretical_loss": 4.154326761248987, + "tokens_seen": 295174144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598696088264794, + "loss": 2.858, + "theoretical_loss": 4.154218268498517, + "tokens_seen": 295239680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045985957873620866, + "loss": 2.9538, + "theoretical_loss": 4.154109806569577, + "tokens_seen": 295305216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598495486459378, + "loss": 3.0794, + "theoretical_loss": 4.154001375446576, + "tokens_seen": 295370752 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459839518555667, + "loss": 3.0819, + "theoretical_loss": 4.15389297511393, + "tokens_seen": 295436288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598294884653962, + "loss": 3.1095, + "theoretical_loss": 4.153784605556069, + "tokens_seen": 295501824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598194583751254, + "loss": 3.2757, + "theoretical_loss": 4.153676266757434, + "tokens_seen": 295567360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045980942828485456, + "loss": 2.7567, + "theoretical_loss": 4.153567958702476, + "tokens_seen": 295632896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597993981945838, + "loss": 3.0827, + "theoretical_loss": 4.153459681375661, + "tokens_seen": 295698432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597893681043129, + "loss": 3.2228, + "theoretical_loss": 4.15335143476146, + "tokens_seen": 295763968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045977933801404216, + "loss": 2.9004, + "theoretical_loss": 4.153243218844362, + "tokens_seen": 295829504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597693079237713, + "loss": 2.9718, + "theoretical_loss": 4.153135033608863, + "tokens_seen": 295895040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597592778335005, + "loss": 2.9733, + "theoretical_loss": 4.153026879039471, + "tokens_seen": 295960576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597492477432297, + "loss": 2.9921, + "theoretical_loss": 4.152918755120708, + "tokens_seen": 296026112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597392176529589, + "loss": 2.9436, + "theoretical_loss": 4.152810661837102, + "tokens_seen": 296091648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045972918756268807, + "loss": 3.1248, + "theoretical_loss": 4.152702599173197, + "tokens_seen": 296157184 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045971915747241725, + "loss": 3.1309, + "theoretical_loss": 4.152594567113546, + "tokens_seen": 296222720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045970912738214643, + "loss": 2.9432, + "theoretical_loss": 4.152486565642714, + "tokens_seen": 296288256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969909729187566, + "loss": 3.2128, + "theoretical_loss": 4.152378594745276, + "tokens_seen": 296353792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596890672016048, + "loss": 3.153, + "theoretical_loss": 4.15227065440582, + "tokens_seen": 296419328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459679037111334, + "loss": 3.1355, + "theoretical_loss": 4.152162744608944, + "tokens_seen": 296484864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 175144, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.142443895339966, + "objective/train/theoretical_loss": 4.152054865339256, + "objective/train/tokens_used": 317010400, + "theoretical_loss": 4.152054865339256, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045966900702106315, + "loss": 3.1106, + "theoretical_loss": 4.152054865339256, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596589769307924, + "loss": 3.0485, + "theoretical_loss": 4.151947016581378, + "tokens_seen": 296615936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045964894684052157, + "loss": 3.0957, + "theoretical_loss": 4.151839198319942, + "tokens_seen": 296681472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045963891675025075, + "loss": 2.7138, + "theoretical_loss": 4.15173141053959, + "tokens_seen": 296747008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045962888665997993, + "loss": 2.8906, + "theoretical_loss": 4.151623653224975, + "tokens_seen": 296812544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045961885656970917, + "loss": 2.9162, + "theoretical_loss": 4.151515926360764, + "tokens_seen": 296878080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596088264794383, + "loss": 2.9349, + "theoretical_loss": 4.151408229931631, + "tokens_seen": 296943616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045959879638916753, + "loss": 3.0348, + "theoretical_loss": 4.151300563922266, + "tokens_seen": 297009152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045958876629889666, + "loss": 3.0585, + "theoretical_loss": 4.151192928317365, + "tokens_seen": 297074688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595787362086259, + "loss": 2.8223, + "theoretical_loss": 4.151085323101638, + "tokens_seen": 297140224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595687061183551, + "loss": 2.8607, + "theoretical_loss": 4.150977748259805, + "tokens_seen": 297205760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045955867602808425, + "loss": 2.8969, + "theoretical_loss": 4.1508702037765985, + "tokens_seen": 297271296 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045954864593781344, + "loss": 2.9414, + "theoretical_loss": 4.150762689636761, + "tokens_seen": 297336832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595386158475426, + "loss": 3.0862, + "theoretical_loss": 4.150655205825045, + "tokens_seen": 297402368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595285857572718, + "loss": 3.018, + "theoretical_loss": 4.150547752326216, + "tokens_seen": 297467904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045951855566700103, + "loss": 2.9801, + "theoretical_loss": 4.15044032912505, + "tokens_seen": 297533440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045950852557673016, + "loss": 3.0379, + "theoretical_loss": 4.150332936206332, + "tokens_seen": 297598976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594984954864594, + "loss": 3.249, + "theoretical_loss": 4.150225573554859, + "tokens_seen": 297664512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594884653961885, + "loss": 3.0122, + "theoretical_loss": 4.150118241155443, + "tokens_seen": 297730048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045947843530591776, + "loss": 3.2498, + "theoretical_loss": 4.1500109389929, + "tokens_seen": 297795584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045946840521564694, + "loss": 2.7565, + "theoretical_loss": 4.1499036670520635, + "tokens_seen": 297861120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594583751253761, + "loss": 3.0607, + "theoretical_loss": 4.149796425317772, + "tokens_seen": 297926656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045944834503510536, + "loss": 3.1681, + "theoretical_loss": 4.149689213774879, + "tokens_seen": 297992192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045943831494483454, + "loss": 3.1897, + "theoretical_loss": 4.149582032408249, + "tokens_seen": 298057728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594282848545637, + "loss": 3.1223, + "theoretical_loss": 4.149474881202755, + "tokens_seen": 298123264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 176258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.113511323928833, + "objective/train/theoretical_loss": 4.1493677601432815, + "objective/train/tokens_used": 318648800, + "theoretical_loss": 4.1493677601432815, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594182547642929, + "loss": 3.0056, + "theoretical_loss": 4.1493677601432815, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594082246740221, + "loss": 3.017, + "theoretical_loss": 4.149260669214725, + "tokens_seen": 298254336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045939819458375126, + "loss": 3.0696, + "theoretical_loss": 4.149153608401994, + "tokens_seen": 298319872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593881644934805, + "loss": 2.9312, + "theoretical_loss": 4.149046577690003, + "tokens_seen": 298385408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593781344032096, + "loss": 2.979, + "theoretical_loss": 4.148939577063683, + "tokens_seen": 298450944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045936810431293886, + "loss": 2.7, + "theoretical_loss": 4.148832606507974, + "tokens_seen": 298516480 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459358074222668, + "loss": 2.9176, + "theoretical_loss": 4.148725666007824, + "tokens_seen": 298582016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593480441323972, + "loss": 2.9862, + "theoretical_loss": 4.1486187555481955, + "tokens_seen": 298647552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593380140421264, + "loss": 2.7821, + "theoretical_loss": 4.14851187511406, + "tokens_seen": 298713088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593279839518556, + "loss": 3.0391, + "theoretical_loss": 4.148405024690402, + "tokens_seen": 298778624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931795386158476, + "loss": 2.9083, + "theoretical_loss": 4.148298204262213, + "tokens_seen": 298844160 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459307923771314, + "loss": 2.8753, + "theoretical_loss": 4.148191413814497, + "tokens_seen": 298909696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592978936810431, + "loss": 2.9916, + "theoretical_loss": 4.148084653332271, + "tokens_seen": 298975232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045928786359077236, + "loss": 3.0754, + "theoretical_loss": 4.14797792280056, + "tokens_seen": 299040768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592778335005015, + "loss": 3.0386, + "theoretical_loss": 4.147871222204401, + "tokens_seen": 299106304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592678034102307, + "loss": 2.9645, + "theoretical_loss": 4.147764551528841, + "tokens_seen": 299171840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592577733199599, + "loss": 2.7558, + "theoretical_loss": 4.147657910758938, + "tokens_seen": 299237376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592477432296891, + "loss": 2.7373, + "theoretical_loss": 4.147551299879762, + "tokens_seen": 299302912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923771313941827, + "loss": 2.8164, + "theoretical_loss": 4.147444718876391, + "tokens_seen": 299368448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045922768304914745, + "loss": 2.8048, + "theoretical_loss": 4.147338167733917, + "tokens_seen": 299433984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045921765295887663, + "loss": 3.0629, + "theoretical_loss": 4.14723164643744, + "tokens_seen": 299499520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045920762286860586, + "loss": 2.948, + "theoretical_loss": 4.1471251549720725, + "tokens_seen": 299565056 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459197592778335, + "loss": 2.9864, + "theoretical_loss": 4.147018693322936, + "tokens_seen": 299630592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591875626880642, + "loss": 2.9198, + "theoretical_loss": 4.146912261475165, + "tokens_seen": 299696128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045917753259779335, + "loss": 3.034, + "theoretical_loss": 4.1468058594139015, + "tokens_seen": 299761664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 176682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.946582078933716, + "objective/train/theoretical_loss": 4.146699487124301, + "objective/train/tokens_used": 320287200, + "theoretical_loss": 4.146699487124301, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591675025075226, + "loss": 3.0244, + "theoretical_loss": 4.146699487124301, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045915747241725177, + "loss": 3.0344, + "theoretical_loss": 4.146593144591527, + "tokens_seen": 299892736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045914744232698095, + "loss": 3.1026, + "theoretical_loss": 4.146486831800758, + "tokens_seen": 299958272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045913741223671013, + "loss": 3.005, + "theoretical_loss": 4.146380548737179, + "tokens_seen": 300023808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045912738214643937, + "loss": 3.1644, + "theoretical_loss": 4.146274295385986, + "tokens_seen": 300089344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591173520561685, + "loss": 2.8718, + "theoretical_loss": 4.146168071732388, + "tokens_seen": 300154880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045910732196589773, + "loss": 3.0563, + "theoretical_loss": 4.146061877761602, + "tokens_seen": 300220416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045909729187562686, + "loss": 3.226, + "theoretical_loss": 4.145955713458856, + "tokens_seen": 300285952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590872617853561, + "loss": 3.0286, + "theoretical_loss": 4.145849578809392, + "tokens_seen": 300351488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590772316950853, + "loss": 2.9362, + "theoretical_loss": 4.145743473798457, + "tokens_seen": 300417024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045906720160481445, + "loss": 3.3005, + "theoretical_loss": 4.145637398411313, + "tokens_seen": 300482560 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045905717151454364, + "loss": 3.1683, + "theoretical_loss": 4.145531352633229, + "tokens_seen": 300548096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590471414242728, + "loss": 2.8494, + "theoretical_loss": 4.14542533644949, + "tokens_seen": 300613632 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459037111334002, + "loss": 2.7638, + "theoretical_loss": 4.145319349845384, + "tokens_seen": 300679168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045902708124373123, + "loss": 3.1229, + "theoretical_loss": 4.145213392806217, + "tokens_seen": 300744704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045901705115346036, + "loss": 2.8249, + "theoretical_loss": 4.145107465317299, + "tokens_seen": 300810240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590070210631896, + "loss": 2.7708, + "theoretical_loss": 4.1450015673639555, + "tokens_seen": 300875776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589969909729187, + "loss": 2.9347, + "theoretical_loss": 4.144895698931521, + "tokens_seen": 300941312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045898696088264796, + "loss": 3.1446, + "theoretical_loss": 4.144789860005337, + "tokens_seen": 301006848 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045897693079237714, + "loss": 2.7626, + "theoretical_loss": 4.14468405057076, + "tokens_seen": 301072384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589669007021063, + "loss": 2.9021, + "theoretical_loss": 4.144578270613157, + "tokens_seen": 301137920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589568706118355, + "loss": 2.9562, + "theoretical_loss": 4.144472520117901, + "tokens_seen": 301203456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045894684052156474, + "loss": 3.0293, + "theoretical_loss": 4.14436679907038, + "tokens_seen": 301268992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045893681043129386, + "loss": 2.9459, + "theoretical_loss": 4.144261107455991, + "tokens_seen": 301334528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589267803410231, + "loss": 2.9452, + "theoretical_loss": 4.1441554452601395, + "tokens_seen": 301400064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 177896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9508297443389893, + "objective/train/theoretical_loss": 4.1440498124682446, + "objective/train/tokens_used": 321925600, + "theoretical_loss": 4.1440498124682446, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589167502507522, + "loss": 3.0381, + "theoretical_loss": 4.1440498124682446, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045890672016048146, + "loss": 3.0254, + "theoretical_loss": 4.143944209065733, + "tokens_seen": 301531136 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045889669007021064, + "loss": 2.948, + "theoretical_loss": 4.143838635038043, + "tokens_seen": 301596672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588866599799398, + "loss": 3.035, + "theoretical_loss": 4.143733090370624, + "tokens_seen": 301662208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458876629889669, + "loss": 2.9586, + "theoretical_loss": 4.143627575048935, + "tokens_seen": 301727744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588665997993982, + "loss": 2.904, + "theoretical_loss": 4.143522089058446, + "tokens_seen": 301793280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045885656970912737, + "loss": 2.9791, + "theoretical_loss": 4.143416632384635, + "tokens_seen": 301858816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588465396188566, + "loss": 2.9168, + "theoretical_loss": 4.143311205012992, + "tokens_seen": 301924352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045883650952858573, + "loss": 2.9416, + "theoretical_loss": 4.14320580692902, + "tokens_seen": 301989888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045882647943831496, + "loss": 3.2419, + "theoretical_loss": 4.143100438118227, + "tokens_seen": 302055424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045881644934804415, + "loss": 3.0354, + "theoretical_loss": 4.142995098566135, + "tokens_seen": 302120960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588064192577733, + "loss": 2.891, + "theoretical_loss": 4.142889788258275, + "tokens_seen": 302186496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587963891675025, + "loss": 3.0476, + "theoretical_loss": 4.142784507180189, + "tokens_seen": 302252032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587863590772317, + "loss": 2.9312, + "theoretical_loss": 4.14267925531743, + "tokens_seen": 302317568 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045877632898696087, + "loss": 3.0632, + "theoretical_loss": 4.142574032655558, + "tokens_seen": 302383104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587662988966901, + "loss": 3.0175, + "theoretical_loss": 4.142468839180148, + "tokens_seen": 302448640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045875626880641923, + "loss": 3.1759, + "theoretical_loss": 4.14236367487678, + "tokens_seen": 302514176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045874623871614847, + "loss": 3.0104, + "theoretical_loss": 4.142258539731047, + "tokens_seen": 302579712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587362086258776, + "loss": 2.8409, + "theoretical_loss": 4.142153433728556, + "tokens_seen": 302645248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045872617853560683, + "loss": 3.0343, + "theoretical_loss": 4.142048356854915, + "tokens_seen": 302710784 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458716148445336, + "loss": 3.1169, + "theoretical_loss": 4.141943309095752, + "tokens_seen": 302776320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587061183550652, + "loss": 2.7164, + "theoretical_loss": 4.141838290436699, + "tokens_seen": 302841856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586960882647944, + "loss": 3.1328, + "theoretical_loss": 4.1417333008633985, + "tokens_seen": 302907392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045868605817452355, + "loss": 2.9251, + "theoretical_loss": 4.141628340361508, + "tokens_seen": 302972928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586760280842528, + "loss": 3.0941, + "theoretical_loss": 4.141523408916689, + "tokens_seen": 303038464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 178251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.023979425430298, + "objective/train/theoretical_loss": 4.141418506514618, + "objective/train/tokens_used": 323564000, + "theoretical_loss": 4.141418506514618, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045866599799398197, + "loss": 3.0323, + "theoretical_loss": 4.141418506514618, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045865596790371115, + "loss": 2.966, + "theoretical_loss": 4.141313633140978, + "tokens_seen": 303169536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045864593781344033, + "loss": 3.0848, + "theoretical_loss": 4.141208788781466, + "tokens_seen": 303235072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045863590772316957, + "loss": 3.0573, + "theoretical_loss": 4.141103973421784, + "tokens_seen": 303300608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586258776328987, + "loss": 3.0507, + "theoretical_loss": 4.14099918704765, + "tokens_seen": 303366144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045861584754262793, + "loss": 2.9278, + "theoretical_loss": 4.140894429644788, + "tokens_seen": 303431680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045860581745235706, + "loss": 3.1005, + "theoretical_loss": 4.1407897011989325, + "tokens_seen": 303497216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585957873620863, + "loss": 3.0779, + "theoretical_loss": 4.1406850016958305, + "tokens_seen": 303562752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585857572718155, + "loss": 2.8597, + "theoretical_loss": 4.140580331121236, + "tokens_seen": 303628288 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045857572718154465, + "loss": 2.8306, + "theoretical_loss": 4.140475689460916, + "tokens_seen": 303693824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045856569709127384, + "loss": 3.0461, + "theoretical_loss": 4.140371076700646, + "tokens_seen": 303759360 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458555667001003, + "loss": 3.0263, + "theoretical_loss": 4.140266492826211, + "tokens_seen": 303824896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585456369107322, + "loss": 3.0801, + "theoretical_loss": 4.140161937823407, + "tokens_seen": 303890432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045853560682046143, + "loss": 2.9155, + "theoretical_loss": 4.140057411678041, + "tokens_seen": 303955968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045852557673019056, + "loss": 2.8495, + "theoretical_loss": 4.139952914375927, + "tokens_seen": 304021504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585155466399198, + "loss": 2.9827, + "theoretical_loss": 4.139848445902892, + "tokens_seen": 304087040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585055165496489, + "loss": 2.9606, + "theoretical_loss": 4.139744006244772, + "tokens_seen": 304152576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045849548645937816, + "loss": 2.8089, + "theoretical_loss": 4.139639595387413, + "tokens_seen": 304218112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045848545636910734, + "loss": 3.0314, + "theoretical_loss": 4.139535213316671, + "tokens_seen": 304283648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584754262788365, + "loss": 2.972, + "theoretical_loss": 4.139430860018411, + "tokens_seen": 304349184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584653961885657, + "loss": 3.0738, + "theoretical_loss": 4.13932653547851, + "tokens_seen": 304414720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045845536609829494, + "loss": 2.8192, + "theoretical_loss": 4.139222239682853, + "tokens_seen": 304480256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045844533600802406, + "loss": 2.805, + "theoretical_loss": 4.139117972617337, + "tokens_seen": 304545792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584353059177533, + "loss": 2.8678, + "theoretical_loss": 4.139013734267866, + "tokens_seen": 304611328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584252758274824, + "loss": 2.9425, + "theoretical_loss": 4.138909524620358, + "tokens_seen": 304676864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 179536, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0331430435180664, + "objective/train/theoretical_loss": 4.138805343660737, + "objective/train/tokens_used": 325202400, + "theoretical_loss": 4.138805343660737, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045841524573721166, + "loss": 2.9569, + "theoretical_loss": 4.138805343660737, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045840521564694084, + "loss": 3.0136, + "theoretical_loss": 4.13870119137494, + "tokens_seen": 304807936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045839518555667, + "loss": 2.7669, + "theoretical_loss": 4.138597067748911, + "tokens_seen": 304873472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583851554663992, + "loss": 2.9702, + "theoretical_loss": 4.138492972768606, + "tokens_seen": 304939008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583751253761284, + "loss": 3.1139, + "theoretical_loss": 4.138388906419991, + "tokens_seen": 305004544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045836509528585757, + "loss": 2.9143, + "theoretical_loss": 4.138284868689041, + "tokens_seen": 305070080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583550651955868, + "loss": 2.7962, + "theoretical_loss": 4.13818085956174, + "tokens_seen": 305135616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045834503510531593, + "loss": 3.0717, + "theoretical_loss": 4.1380768790240845, + "tokens_seen": 305201152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045833500501504516, + "loss": 2.8641, + "theoretical_loss": 4.137972927062078, + "tokens_seen": 305266688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045832497492477435, + "loss": 2.9519, + "theoretical_loss": 4.137869003661737, + "tokens_seen": 305332224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583149448345035, + "loss": 3.1127, + "theoretical_loss": 4.137765108809084, + "tokens_seen": 305397760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583049147442327, + "loss": 2.9791, + "theoretical_loss": 4.1376612424901555, + "tokens_seen": 305463296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582948846539619, + "loss": 2.9104, + "theoretical_loss": 4.137557404690993, + "tokens_seen": 305528832 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045828485456369107, + "loss": 3.0647, + "theoretical_loss": 4.137453595397653, + "tokens_seen": 305594368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582748244734203, + "loss": 2.9847, + "theoretical_loss": 4.137349814596198, + "tokens_seen": 305659904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045826479438314943, + "loss": 3.0683, + "theoretical_loss": 4.137246062272702, + "tokens_seen": 305725440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045825476429287867, + "loss": 2.9846, + "theoretical_loss": 4.137142338413248, + "tokens_seen": 305790976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582447342026078, + "loss": 2.7466, + "theoretical_loss": 4.137038643003931, + "tokens_seen": 305856512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045823470411233703, + "loss": 2.9283, + "theoretical_loss": 4.136934976030852, + "tokens_seen": 305922048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582246740220662, + "loss": 3.0245, + "theoretical_loss": 4.136831337480125, + "tokens_seen": 305987584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582146439317954, + "loss": 2.8523, + "theoretical_loss": 4.13672772733787, + "tokens_seen": 306053120 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045820461384152457, + "loss": 2.8644, + "theoretical_loss": 4.136624145590222, + "tokens_seen": 306118656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045819458375125375, + "loss": 3.1607, + "theoretical_loss": 4.136520592223322, + "tokens_seen": 306184192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045818455366098294, + "loss": 2.9101, + "theoretical_loss": 4.136417067223322, + "tokens_seen": 306249728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045817452357071217, + "loss": 3.1469, + "theoretical_loss": 4.136313570576383, + "tokens_seen": 306315264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 180132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.187363624572754, + "objective/train/theoretical_loss": 4.136210102268676, + "objective/train/tokens_used": 326840800, + "theoretical_loss": 4.136210102268676, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581644934804413, + "loss": 3.0491, + "theoretical_loss": 4.136210102268676, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045815446339017053, + "loss": 3.2312, + "theoretical_loss": 4.136106662286383, + "tokens_seen": 306446336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581444332998997, + "loss": 3.1742, + "theoretical_loss": 4.136003250615693, + "tokens_seen": 306511872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581344032096289, + "loss": 2.983, + "theoretical_loss": 4.135899867242808, + "tokens_seen": 306577408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581243731193581, + "loss": 3.1178, + "theoretical_loss": 4.135796512153934, + "tokens_seen": 306642944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045811434302908726, + "loss": 2.8261, + "theoretical_loss": 4.135693185335295, + "tokens_seen": 306708480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045810431293881644, + "loss": 3.001, + "theoretical_loss": 4.135589886773119, + "tokens_seen": 306774016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580942828485457, + "loss": 2.9546, + "theoretical_loss": 4.135486616453644, + "tokens_seen": 306839552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580842527582748, + "loss": 3.0585, + "theoretical_loss": 4.135383374363119, + "tokens_seen": 306905088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045807422266800404, + "loss": 3.023, + "theoretical_loss": 4.1352801604878024, + "tokens_seen": 306970624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045806419257773316, + "loss": 2.954, + "theoretical_loss": 4.135176974813961, + "tokens_seen": 307036160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580541624874624, + "loss": 3.259, + "theoretical_loss": 4.135073817327872, + "tokens_seen": 307101696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580441323971916, + "loss": 3.053, + "theoretical_loss": 4.134970688015824, + "tokens_seen": 307167232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045803410230692076, + "loss": 2.8994, + "theoretical_loss": 4.134867586864112, + "tokens_seen": 307232768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045802407221664994, + "loss": 3.019, + "theoretical_loss": 4.134764513859043, + "tokens_seen": 307298304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580140421263791, + "loss": 2.7869, + "theoretical_loss": 4.134661468986932, + "tokens_seen": 307363840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580040120361083, + "loss": 2.8573, + "theoretical_loss": 4.1345584522341055, + "tokens_seen": 307429376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045799398194583754, + "loss": 2.8166, + "theoretical_loss": 4.134455463586898, + "tokens_seen": 307494912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045798395185556667, + "loss": 2.6522, + "theoretical_loss": 4.134352503031652, + "tokens_seen": 307560448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579739217652959, + "loss": 3.0032, + "theoretical_loss": 4.134249570554724, + "tokens_seen": 307625984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579638916750251, + "loss": 2.8976, + "theoretical_loss": 4.134146666142476, + "tokens_seen": 307691520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045795386158475426, + "loss": 2.7396, + "theoretical_loss": 4.134043789781281, + "tokens_seen": 307757056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579438314944835, + "loss": 3.0013, + "theoretical_loss": 4.1339409414575226, + "tokens_seen": 307822592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579338014042126, + "loss": 2.9292, + "theoretical_loss": 4.133838121157592, + "tokens_seen": 307888128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045792377131394186, + "loss": 3.2311, + "theoretical_loss": 4.133735328867892, + "tokens_seen": 307953664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 181325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.002859354019165, + "objective/train/theoretical_loss": 4.133632564574831, + "objective/train/tokens_used": 328479200, + "theoretical_loss": 4.133632564574831, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045791374122367104, + "loss": 2.918, + "theoretical_loss": 4.133632564574831, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579037111334002, + "loss": 2.8607, + "theoretical_loss": 4.133529828264831, + "tokens_seen": 308084736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578936810431294, + "loss": 2.9817, + "theoretical_loss": 4.133427119924323, + "tokens_seen": 308150272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578836509528586, + "loss": 3.1387, + "theoretical_loss": 4.133324439539745, + "tokens_seen": 308215808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045787362086258777, + "loss": 3.2184, + "theoretical_loss": 4.133221787097546, + "tokens_seen": 308281344 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457863590772317, + "loss": 2.8897, + "theoretical_loss": 4.133119162584185, + "tokens_seen": 308346880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045785356068204613, + "loss": 3.0767, + "theoretical_loss": 4.1330165659861295, + "tokens_seen": 308412416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045784353059177536, + "loss": 2.9042, + "theoretical_loss": 4.132913997289856, + "tokens_seen": 308477952 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783350050150455, + "loss": 2.9545, + "theoretical_loss": 4.1328114564818526, + "tokens_seen": 308543488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578234704112337, + "loss": 2.8544, + "theoretical_loss": 4.132708943548614, + "tokens_seen": 308609024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578134403209629, + "loss": 3.1091, + "theoretical_loss": 4.132606458476647, + "tokens_seen": 308674560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578034102306921, + "loss": 2.9141, + "theoretical_loss": 4.132504001252465, + "tokens_seen": 308740096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045779338014042127, + "loss": 2.9562, + "theoretical_loss": 4.132401571862593, + "tokens_seen": 308805632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577833500501505, + "loss": 2.9368, + "theoretical_loss": 4.132299170293564, + "tokens_seen": 308871168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045777331995987963, + "loss": 2.8221, + "theoretical_loss": 4.132196796531922, + "tokens_seen": 308936704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045776328986960887, + "loss": 2.9433, + "theoretical_loss": 4.132094450564218, + "tokens_seen": 309002240 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457753259779338, + "loss": 2.8562, + "theoretical_loss": 4.1319921323770155, + "tokens_seen": 309067776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045774322968906723, + "loss": 3.0209, + "theoretical_loss": 4.131889841956884, + "tokens_seen": 309133312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577331995987964, + "loss": 3.009, + "theoretical_loss": 4.131787579290405, + "tokens_seen": 309198848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577231695085256, + "loss": 2.7863, + "theoretical_loss": 4.1316853443641675, + "tokens_seen": 309264384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577131394182548, + "loss": 3.0576, + "theoretical_loss": 4.13158313716477, + "tokens_seen": 309329920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045770310932798395, + "loss": 2.9071, + "theoretical_loss": 4.131480957678823, + "tokens_seen": 309395456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045769307923771314, + "loss": 2.9949, + "theoretical_loss": 4.1313788058929415, + "tokens_seen": 309460992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045768304914744237, + "loss": 2.8319, + "theoretical_loss": 4.131276681793754, + "tokens_seen": 309526528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576730190571715, + "loss": 2.7952, + "theoretical_loss": 4.131174585367896, + "tokens_seen": 309592064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 182674, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9335567951202393, + "objective/train/theoretical_loss": 4.131072516602014, + "objective/train/tokens_used": 330117600, + "theoretical_loss": 4.131072516602014, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045766298896690073, + "loss": 2.9291, + "theoretical_loss": 4.131072516602014, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576529588766299, + "loss": 2.8759, + "theoretical_loss": 4.130970475482761, + "tokens_seen": 309723136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576429287863591, + "loss": 2.7693, + "theoretical_loss": 4.130868461996803, + "tokens_seen": 309788672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576328986960883, + "loss": 3.2834, + "theoretical_loss": 4.1307664761308125, + "tokens_seen": 309854208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045762286860581746, + "loss": 2.7814, + "theoretical_loss": 4.130664517871471, + "tokens_seen": 309919744 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045761283851554664, + "loss": 3.364, + "theoretical_loss": 4.130562587205473, + "tokens_seen": 309985280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576028084252759, + "loss": 2.8317, + "theoretical_loss": 4.130460684119516, + "tokens_seen": 310050816 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457592778335005, + "loss": 2.9605, + "theoretical_loss": 4.130358808600313, + "tokens_seen": 310116352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045758274824473424, + "loss": 3.1335, + "theoretical_loss": 4.130256960634582, + "tokens_seen": 310181888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045757271815446336, + "loss": 3.0687, + "theoretical_loss": 4.130155140209052, + "tokens_seen": 310247424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575626880641926, + "loss": 2.9797, + "theoretical_loss": 4.130053347310461, + "tokens_seen": 310312960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575526579739218, + "loss": 3.294, + "theoretical_loss": 4.129951581925555, + "tokens_seen": 310378496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045754262788365096, + "loss": 2.9568, + "theoretical_loss": 4.129849844041091, + "tokens_seen": 310444032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045753259779338014, + "loss": 2.941, + "theoretical_loss": 4.129748133643835, + "tokens_seen": 310509568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575225677031093, + "loss": 3.0219, + "theoretical_loss": 4.129646450720561, + "tokens_seen": 310575104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575125376128385, + "loss": 2.9852, + "theoretical_loss": 4.129544795258052, + "tokens_seen": 310640640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045750250752256774, + "loss": 2.8331, + "theoretical_loss": 4.129443167243101, + "tokens_seen": 310706176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045749247743229687, + "loss": 3.061, + "theoretical_loss": 4.129341566662511, + "tokens_seen": 310771712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574824473420261, + "loss": 2.967, + "theoretical_loss": 4.1292399935030915, + "tokens_seen": 310837248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574724172517553, + "loss": 3.0105, + "theoretical_loss": 4.129138447751664, + "tokens_seen": 310902784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045746238716148446, + "loss": 2.9289, + "theoretical_loss": 4.129036929395058, + "tokens_seen": 310968320 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045745235707121364, + "loss": 3.0223, + "theoretical_loss": 4.12893543842011, + "tokens_seen": 311033856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574423269809428, + "loss": 2.9751, + "theoretical_loss": 4.12883397481367, + "tokens_seen": 311099392 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457432296890672, + "loss": 2.8895, + "theoretical_loss": 4.128732538562592, + "tokens_seen": 311164928 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045742226680040124, + "loss": 2.7377, + "theoretical_loss": 4.128631129653744, + "tokens_seen": 311230464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 183229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0134546756744385, + "objective/train/theoretical_loss": 4.128529748073999, + "objective/train/tokens_used": 331756000, + "theoretical_loss": 4.128529748073999, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045741223671013037, + "loss": 2.6315, + "theoretical_loss": 4.128529748073999, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574022066198596, + "loss": 3.0727, + "theoretical_loss": 4.128428393810242, + "tokens_seen": 311361536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739217652958873, + "loss": 3.0927, + "theoretical_loss": 4.128327066849365, + "tokens_seen": 311427072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045738214643931797, + "loss": 2.984, + "theoretical_loss": 4.12822576717827, + "tokens_seen": 311492608 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045737211634904715, + "loss": 3.1219, + "theoretical_loss": 4.128124494783869, + "tokens_seen": 311558144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045736208625877633, + "loss": 2.8423, + "theoretical_loss": 4.12802324965308, + "tokens_seen": 311623680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573520561685055, + "loss": 3.034, + "theoretical_loss": 4.127922031772834, + "tokens_seen": 311689216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045734202607823475, + "loss": 3.0936, + "theoretical_loss": 4.127820841130068, + "tokens_seen": 311754752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045733199598796387, + "loss": 2.8057, + "theoretical_loss": 4.127719677711729, + "tokens_seen": 311820288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573219658976931, + "loss": 2.8598, + "theoretical_loss": 4.1276185415047735, + "tokens_seen": 311885824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045731193580742223, + "loss": 2.8944, + "theoretical_loss": 4.127517432496166, + "tokens_seen": 311951360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045730190571715147, + "loss": 2.9157, + "theoretical_loss": 4.127416350672879, + "tokens_seen": 312016896 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045729187562688065, + "loss": 2.938, + "theoretical_loss": 4.1273152960219, + "tokens_seen": 312082432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045728184553660983, + "loss": 3.0826, + "theoretical_loss": 4.127214268530216, + "tokens_seen": 312147968 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457271815446339, + "loss": 3.0073, + "theoretical_loss": 4.12711326818483, + "tokens_seen": 312213504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572617853560682, + "loss": 3.0567, + "theoretical_loss": 4.127012294972753, + "tokens_seen": 312279040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572517552657974, + "loss": 3.169, + "theoretical_loss": 4.126911348881, + "tokens_seen": 312344576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572417251755266, + "loss": 2.9469, + "theoretical_loss": 4.126810429896603, + "tokens_seen": 312410112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045723169508525574, + "loss": 2.9813, + "theoretical_loss": 4.126709538006596, + "tokens_seen": 312475648 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457221664994985, + "loss": 2.934, + "theoretical_loss": 4.1266086731980245, + "tokens_seen": 312541184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572116349047141, + "loss": 3.2086, + "theoretical_loss": 4.126507835457945, + "tokens_seen": 312606720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045720160481444334, + "loss": 2.7577, + "theoretical_loss": 4.126407024773418, + "tokens_seen": 312672256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045719157472417257, + "loss": 2.9729, + "theoretical_loss": 4.126306241131519, + "tokens_seen": 312737792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571815446339017, + "loss": 2.9627, + "theoretical_loss": 4.126205484519327, + "tokens_seen": 312803328 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045717151454363093, + "loss": 3.0685, + "theoretical_loss": 4.126104754923931, + "tokens_seen": 312868864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 184180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2469050884246826, + "objective/train/theoretical_loss": 4.126004052332432, + "objective/train/tokens_used": 333394400, + "theoretical_loss": 4.126004052332432, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571614844533601, + "loss": 3.0363, + "theoretical_loss": 4.126004052332432, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571514543630893, + "loss": 2.9679, + "theoretical_loss": 4.125903376731936, + "tokens_seen": 312999936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571414242728185, + "loss": 2.8487, + "theoretical_loss": 4.125802728109562, + "tokens_seen": 313065472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045713139418254766, + "loss": 2.8367, + "theoretical_loss": 4.125702106452433, + "tokens_seen": 313131008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045712136409227684, + "loss": 2.9977, + "theoretical_loss": 4.125601511747685, + "tokens_seen": 313196544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571113340020061, + "loss": 3.0608, + "theoretical_loss": 4.125500943982459, + "tokens_seen": 313262080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571013039117352, + "loss": 2.8297, + "theoretical_loss": 4.125400403143908, + "tokens_seen": 313327616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045709127382146444, + "loss": 3.052, + "theoretical_loss": 4.125299889219194, + "tokens_seen": 313393152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045708124373119356, + "loss": 3.0677, + "theoretical_loss": 4.125199402195486, + "tokens_seen": 313458688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570712136409228, + "loss": 2.9362, + "theoretical_loss": 4.12509894205996, + "tokens_seen": 313524224 + }, + { + "epoch": 0.1, + "learning_rate": 0.000457061183550652, + "loss": 2.9709, + "theoretical_loss": 4.124998508799806, + "tokens_seen": 313589760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045705115346038116, + "loss": 3.04, + "theoretical_loss": 4.124898102402218, + "tokens_seen": 313655296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045704112337011034, + "loss": 3.0373, + "theoretical_loss": 4.124797722854401, + "tokens_seen": 313720832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570310932798395, + "loss": 2.8581, + "theoretical_loss": 4.12469737014357, + "tokens_seen": 313786368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570210631895687, + "loss": 3.0926, + "theoretical_loss": 4.124597044256945, + "tokens_seen": 313851904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045701103309929794, + "loss": 3.0979, + "theoretical_loss": 4.124496745181759, + "tokens_seen": 313917440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045700100300902707, + "loss": 2.9877, + "theoretical_loss": 4.12439647290525, + "tokens_seen": 313982976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569909729187563, + "loss": 3.2337, + "theoretical_loss": 4.124296227414667, + "tokens_seen": 314048512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569809428284855, + "loss": 2.9723, + "theoretical_loss": 4.124196008697267, + "tokens_seen": 314114048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045697091273821466, + "loss": 3.2751, + "theoretical_loss": 4.124095816740317, + "tokens_seen": 314179584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045696088264794384, + "loss": 2.9168, + "theoretical_loss": 4.123995651531089, + "tokens_seen": 314245120 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456950852557673, + "loss": 2.9351, + "theoretical_loss": 4.123895513056869, + "tokens_seen": 314310656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569408224674022, + "loss": 3.0289, + "theoretical_loss": 4.123795401304948, + "tokens_seen": 314376192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045693079237713144, + "loss": 2.9276, + "theoretical_loss": 4.123695316262626, + "tokens_seen": 314441728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045692076228686057, + "loss": 3.3239, + "theoretical_loss": 4.123595257917214, + "tokens_seen": 314507264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 184828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.897738218307495, + "objective/train/theoretical_loss": 4.123495226256027, + "objective/train/tokens_used": 335032800, + "theoretical_loss": 4.123495226256027, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569107321965898, + "loss": 2.8664, + "theoretical_loss": 4.123495226256027, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045690070210631893, + "loss": 2.9238, + "theoretical_loss": 4.123395221266396, + "tokens_seen": 314638336 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045689067201604817, + "loss": 3.0188, + "theoretical_loss": 4.123295242935653, + "tokens_seen": 314703872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045688064192577735, + "loss": 2.9396, + "theoretical_loss": 4.123195291251142, + "tokens_seen": 314769408 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045687061183550653, + "loss": 3.1508, + "theoretical_loss": 4.123095366200217, + "tokens_seen": 314834944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568605817452357, + "loss": 2.9989, + "theoretical_loss": 4.122995467770239, + "tokens_seen": 314900480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045685055165496495, + "loss": 2.9205, + "theoretical_loss": 4.122895595948577, + "tokens_seen": 314966016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045684052156469407, + "loss": 2.8574, + "theoretical_loss": 4.122795750722612, + "tokens_seen": 315031552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568304914744233, + "loss": 2.7756, + "theoretical_loss": 4.122695932079728, + "tokens_seen": 315097088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045682046138415243, + "loss": 3.1906, + "theoretical_loss": 4.122596140007323, + "tokens_seen": 315162624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045681043129388167, + "loss": 2.9296, + "theoretical_loss": 4.122496374492799, + "tokens_seen": 315228160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045680040120361085, + "loss": 2.6739, + "theoretical_loss": 4.122396635523571, + "tokens_seen": 315293696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045679037111334003, + "loss": 2.8686, + "theoretical_loss": 4.12229692308706, + "tokens_seen": 315359232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567803410230692, + "loss": 3.2873, + "theoretical_loss": 4.122197237170695, + "tokens_seen": 315424768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567703109327984, + "loss": 3.0861, + "theoretical_loss": 4.122097577761917, + "tokens_seen": 315490304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567602808425276, + "loss": 3.1712, + "theoretical_loss": 4.121997944848171, + "tokens_seen": 315555840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567502507522568, + "loss": 3.1188, + "theoretical_loss": 4.121898338416912, + "tokens_seen": 315621376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045674022066198594, + "loss": 2.9156, + "theoretical_loss": 4.121798758455608, + "tokens_seen": 315686912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567301905717152, + "loss": 2.9989, + "theoretical_loss": 4.121699204951728, + "tokens_seen": 315752448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567201604814443, + "loss": 2.6808, + "theoretical_loss": 4.121599677892755, + "tokens_seen": 315817984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045671013039117354, + "loss": 2.9749, + "theoretical_loss": 4.121500177266178, + "tokens_seen": 315883520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567001003009027, + "loss": 2.9981, + "theoretical_loss": 4.121400703059497, + "tokens_seen": 315949056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566900702106319, + "loss": 3.0599, + "theoretical_loss": 4.121301255260217, + "tokens_seen": 316014592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566800401203611, + "loss": 2.9343, + "theoretical_loss": 4.121201833855856, + "tokens_seen": 316080128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566700100300903, + "loss": 3.0488, + "theoretical_loss": 4.121102438833935, + "tokens_seen": 316145664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 186091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.028201103210449, + "objective/train/theoretical_loss": 4.121003070181987, + "objective/train/tokens_used": 336671200, + "theoretical_loss": 4.121003070181987, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045665997993981944, + "loss": 3.0888, + "theoretical_loss": 4.121003070181987, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566499498495487, + "loss": 2.8606, + "theoretical_loss": 4.120903727887554, + "tokens_seen": 316276736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566399197592778, + "loss": 2.8707, + "theoretical_loss": 4.120804411938185, + "tokens_seen": 316342272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045662988966900704, + "loss": 3.1801, + "theoretical_loss": 4.120705122321437, + "tokens_seen": 316407808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566198595787362, + "loss": 2.9392, + "theoretical_loss": 4.1206058590248755, + "tokens_seen": 316473344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566098294884654, + "loss": 3.0525, + "theoretical_loss": 4.120506622036077, + "tokens_seen": 316538880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565997993981946, + "loss": 2.8932, + "theoretical_loss": 4.120407411342623, + "tokens_seen": 316604416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045658976930792376, + "loss": 3.0008, + "theoretical_loss": 4.1203082269321065, + "tokens_seen": 316669952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045657973921765294, + "loss": 2.9041, + "theoretical_loss": 4.120209068792127, + "tokens_seen": 316735488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565697091273822, + "loss": 2.8144, + "theoretical_loss": 4.120109936910291, + "tokens_seen": 316801024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565596790371113, + "loss": 2.9752, + "theoretical_loss": 4.120010831274218, + "tokens_seen": 316866560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045654964894684054, + "loss": 2.9861, + "theoretical_loss": 4.119911751871531, + "tokens_seen": 316932096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045653961885656967, + "loss": 3.1201, + "theoretical_loss": 4.1198126986898655, + "tokens_seen": 316997632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565295887662989, + "loss": 3.2564, + "theoretical_loss": 4.119713671716862, + "tokens_seen": 317063168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565195586760281, + "loss": 3.0303, + "theoretical_loss": 4.11961467094017, + "tokens_seen": 317128704 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045650952858575727, + "loss": 3.0001, + "theoretical_loss": 4.119515696347451, + "tokens_seen": 317194240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045649949849548645, + "loss": 3.1713, + "theoretical_loss": 4.119416747926369, + "tokens_seen": 317259776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564894684052157, + "loss": 2.9097, + "theoretical_loss": 4.119317825664601, + "tokens_seen": 317325312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564794383149448, + "loss": 3.0256, + "theoretical_loss": 4.119218929549831, + "tokens_seen": 317390848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045646940822467405, + "loss": 2.9957, + "theoretical_loss": 4.119120059569751, + "tokens_seen": 317456384 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045645937813440317, + "loss": 3.1022, + "theoretical_loss": 4.11902121571206, + "tokens_seen": 317521920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564493480441324, + "loss": 2.8814, + "theoretical_loss": 4.118922397964468, + "tokens_seen": 317587456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045643931795386164, + "loss": 2.8242, + "theoretical_loss": 4.118823606314692, + "tokens_seen": 317652992 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045642928786359077, + "loss": 3.1631, + "theoretical_loss": 4.118724840750456, + "tokens_seen": 317718528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045641925777332, + "loss": 3.0526, + "theoretical_loss": 4.118626101259496, + "tokens_seen": 317784064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 187059, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0989761352539062, + "objective/train/theoretical_loss": 4.118527387829552, + "objective/train/tokens_used": 338309600, + "theoretical_loss": 4.118527387829552, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045640922768304913, + "loss": 3.1599, + "theoretical_loss": 4.118527387829552, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045639919759277837, + "loss": 2.9083, + "theoretical_loss": 4.118428700448375, + "tokens_seen": 317915136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045638916750250755, + "loss": 2.7942, + "theoretical_loss": 4.118330039103723, + "tokens_seen": 317980672 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045637913741223673, + "loss": 2.8139, + "theoretical_loss": 4.118231403783365, + "tokens_seen": 318046208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563691073219659, + "loss": 3.037, + "theoretical_loss": 4.118132794475073, + "tokens_seen": 318111744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045635907723169515, + "loss": 3.0491, + "theoretical_loss": 4.118034211166632, + "tokens_seen": 318177280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045634904714142427, + "loss": 2.8304, + "theoretical_loss": 4.117935653845834, + "tokens_seen": 318242816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563390170511535, + "loss": 2.9257, + "theoretical_loss": 4.117837122500477, + "tokens_seen": 318308352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045632898696088263, + "loss": 3.1308, + "theoretical_loss": 4.117738617118372, + "tokens_seen": 318373888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045631895687061187, + "loss": 2.8832, + "theoretical_loss": 4.1176401376873315, + "tokens_seen": 318439424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045630892678034105, + "loss": 3.0881, + "theoretical_loss": 4.117541684195183, + "tokens_seen": 318504960 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045629889669007023, + "loss": 3.0655, + "theoretical_loss": 4.117443256629758, + "tokens_seen": 318570496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562888665997994, + "loss": 3.2203, + "theoretical_loss": 4.1173448549788985, + "tokens_seen": 318636032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562788365095286, + "loss": 2.8932, + "theoretical_loss": 4.117246479230452, + "tokens_seen": 318701568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562688064192578, + "loss": 3.1329, + "theoretical_loss": 4.117148129372277, + "tokens_seen": 318767104 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456258776328987, + "loss": 3.0318, + "theoretical_loss": 4.117049805392239, + "tokens_seen": 318832640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045624874623871614, + "loss": 3.0924, + "theoretical_loss": 4.116951507278211, + "tokens_seen": 318898176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562387161484454, + "loss": 3.2415, + "theoretical_loss": 4.116853235018075, + "tokens_seen": 318963712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562286860581745, + "loss": 2.8784, + "theoretical_loss": 4.116754988599722, + "tokens_seen": 319029248 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045621865596790374, + "loss": 2.9519, + "theoretical_loss": 4.11665676801105, + "tokens_seen": 319094784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562086258776329, + "loss": 3.0788, + "theoretical_loss": 4.116558573239965, + "tokens_seen": 319160320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561985957873621, + "loss": 2.9773, + "theoretical_loss": 4.116460404274381, + "tokens_seen": 319225856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561885656970913, + "loss": 2.8958, + "theoretical_loss": 4.116362261102221, + "tokens_seen": 319291392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561785356068205, + "loss": 3.1937, + "theoretical_loss": 4.116264143711416, + "tokens_seen": 319356928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045616850551654964, + "loss": 3.0717, + "theoretical_loss": 4.1161660520899055, + "tokens_seen": 319422464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 187800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.174799680709839, + "objective/train/theoretical_loss": 4.116067986225635, + "objective/train/tokens_used": 339948000, + "theoretical_loss": 4.116067986225635, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561584754262789, + "loss": 3.0015, + "theoretical_loss": 4.116067986225635, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456148445336008, + "loss": 2.8217, + "theoretical_loss": 4.11596994610656, + "tokens_seen": 319553536 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045613841524573724, + "loss": 3.3545, + "theoretical_loss": 4.115871931720645, + "tokens_seen": 319619072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561283851554664, + "loss": 2.7344, + "theoretical_loss": 4.11577394305586, + "tokens_seen": 319684608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561183550651956, + "loss": 2.8487, + "theoretical_loss": 4.115675980100185, + "tokens_seen": 319750144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561083249749248, + "loss": 2.9135, + "theoretical_loss": 4.115578042841607, + "tokens_seen": 319815680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045609829488465396, + "loss": 2.9709, + "theoretical_loss": 4.115480131268122, + "tokens_seen": 319881216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045608826479438314, + "loss": 2.9644, + "theoretical_loss": 4.1153822453677344, + "tokens_seen": 319946752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560782347041124, + "loss": 2.9821, + "theoretical_loss": 4.115284385128454, + "tokens_seen": 320012288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560682046138415, + "loss": 3.1831, + "theoretical_loss": 4.115186550538302, + "tokens_seen": 320077824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045605817452357074, + "loss": 3.0413, + "theoretical_loss": 4.115088741585305, + "tokens_seen": 320143360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045604814443329987, + "loss": 2.9108, + "theoretical_loss": 4.114990958257501, + "tokens_seen": 320208896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560381143430291, + "loss": 3.2611, + "theoretical_loss": 4.11489320054293, + "tokens_seen": 320274432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560280842527583, + "loss": 2.6443, + "theoretical_loss": 4.114795468429648, + "tokens_seen": 320339968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045601805416248747, + "loss": 3.2413, + "theoretical_loss": 4.114697761905712, + "tokens_seen": 320405504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045600802407221665, + "loss": 3.2062, + "theoretical_loss": 4.1146000809591925, + "tokens_seen": 320471040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559979939819459, + "loss": 3.2569, + "theoretical_loss": 4.114502425578163, + "tokens_seen": 320536576 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455987963891675, + "loss": 3.047, + "theoretical_loss": 4.1144047957507075, + "tokens_seen": 320602112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045597793380140425, + "loss": 2.9388, + "theoretical_loss": 4.114307191464919, + "tokens_seen": 320667648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045596790371113337, + "loss": 3.0027, + "theoretical_loss": 4.114209612708898, + "tokens_seen": 320733184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559578736208626, + "loss": 2.8668, + "theoretical_loss": 4.1141120594707505, + "tokens_seen": 320798720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559478435305918, + "loss": 3.1505, + "theoretical_loss": 4.114014531738593, + "tokens_seen": 320864256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045593781344032097, + "loss": 3.0058, + "theoretical_loss": 4.113917029500549, + "tokens_seen": 320929792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045592778335005015, + "loss": 3.057, + "theoretical_loss": 4.113819552744752, + "tokens_seen": 320995328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045591775325977933, + "loss": 2.9634, + "theoretical_loss": 4.11372210145934, + "tokens_seen": 321060864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 188718, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.97586989402771, + "objective/train/theoretical_loss": 4.11362467563246, + "objective/train/tokens_used": 341586400, + "theoretical_loss": 4.11362467563246, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559077231695085, + "loss": 3.0639, + "theoretical_loss": 4.11362467563246, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045589769307923775, + "loss": 3.2388, + "theoretical_loss": 4.1135272752522685, + "tokens_seen": 321191936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558876629889669, + "loss": 3.1461, + "theoretical_loss": 4.1134299003069295, + "tokens_seen": 321257472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558776328986961, + "loss": 3.0283, + "theoretical_loss": 4.113332550784613, + "tokens_seen": 321323008 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045586760280842524, + "loss": 2.9909, + "theoretical_loss": 4.113235226673499, + "tokens_seen": 321388544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558575727181545, + "loss": 3.0179, + "theoretical_loss": 4.113137927961776, + "tokens_seen": 321454080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045584754262788365, + "loss": 2.8515, + "theoretical_loss": 4.113040654637637, + "tokens_seen": 321519616 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045583751253761284, + "loss": 2.755, + "theoretical_loss": 4.112943406689286, + "tokens_seen": 321585152 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455827482447342, + "loss": 2.9629, + "theoretical_loss": 4.112846184104933, + "tokens_seen": 321650688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045581745235707125, + "loss": 2.9968, + "theoretical_loss": 4.112748986872798, + "tokens_seen": 321716224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558074222668004, + "loss": 3.2506, + "theoretical_loss": 4.112651814981108, + "tokens_seen": 321781760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557973921765296, + "loss": 3.103, + "theoretical_loss": 4.112554668418095, + "tokens_seen": 321847296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045578736208625874, + "loss": 3.1271, + "theoretical_loss": 4.112457547172005, + "tokens_seen": 321912832 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455777331995988, + "loss": 3.2188, + "theoretical_loss": 4.112360451231083, + "tokens_seen": 321978368 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045576730190571716, + "loss": 3.0657, + "theoretical_loss": 4.112263380583593, + "tokens_seen": 322043904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045575727181544634, + "loss": 3.1376, + "theoretical_loss": 4.112166335217797, + "tokens_seen": 322109440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557472417251755, + "loss": 2.8802, + "theoretical_loss": 4.112069315121969, + "tokens_seen": 322174976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557372116349047, + "loss": 3.0532, + "theoretical_loss": 4.111972320284392, + "tokens_seen": 322240512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557271815446339, + "loss": 2.9202, + "theoretical_loss": 4.111875350693354, + "tokens_seen": 322306048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557171514543631, + "loss": 3.1109, + "theoretical_loss": 4.111778406337153, + "tokens_seen": 322371584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045570712136409224, + "loss": 2.8451, + "theoretical_loss": 4.111681487204094, + "tokens_seen": 322437120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556970912738215, + "loss": 2.957, + "theoretical_loss": 4.111584593282489, + "tokens_seen": 322502656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556870611835507, + "loss": 3.2971, + "theoretical_loss": 4.111487724560659, + "tokens_seen": 322568192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045567703109327984, + "loss": 3.227, + "theoretical_loss": 4.111390881026932, + "tokens_seen": 322633728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556670010030091, + "loss": 3.1005, + "theoretical_loss": 4.111294062669645, + "tokens_seen": 322699264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 189140, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8794915676116943, + "objective/train/theoretical_loss": 4.111197269477141, + "objective/train/tokens_used": 343224800, + "theoretical_loss": 4.111197269477141, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556569709127382, + "loss": 2.9512, + "theoretical_loss": 4.111197269477141, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045564694082246744, + "loss": 3.0384, + "theoretical_loss": 4.111100501437772, + "tokens_seen": 322830336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556369107321966, + "loss": 2.8792, + "theoretical_loss": 4.111003758539896, + "tokens_seen": 322895872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556268806419258, + "loss": 3.1337, + "theoretical_loss": 4.110907040771883, + "tokens_seen": 322961408 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455616850551655, + "loss": 3.186, + "theoretical_loss": 4.110810348122105, + "tokens_seen": 323026944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045560682046138416, + "loss": 2.9821, + "theoretical_loss": 4.110713680578947, + "tokens_seen": 323092480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045559679037111334, + "loss": 3.2378, + "theoretical_loss": 4.110617038130797, + "tokens_seen": 323158016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555867602808426, + "loss": 2.6982, + "theoretical_loss": 4.110520420766054, + "tokens_seen": 323223552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555767301905717, + "loss": 3.3153, + "theoretical_loss": 4.110423828473126, + "tokens_seen": 323289088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045556670010030094, + "loss": 3.0101, + "theoretical_loss": 4.110327261240422, + "tokens_seen": 323354624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045555667001003007, + "loss": 3.1061, + "theoretical_loss": 4.110230719056367, + "tokens_seen": 323420160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555466399197593, + "loss": 3.1276, + "theoretical_loss": 4.110134201909388, + "tokens_seen": 323485696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555366098294885, + "loss": 3.2466, + "theoretical_loss": 4.110037709787923, + "tokens_seen": 323551232 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045552657973921767, + "loss": 3.347, + "theoretical_loss": 4.109941242680414, + "tokens_seen": 323616768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045551654964894685, + "loss": 3.1481, + "theoretical_loss": 4.109844800575315, + "tokens_seen": 323682304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555065195586761, + "loss": 2.9884, + "theoretical_loss": 4.109748383461085, + "tokens_seen": 323747840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554964894684052, + "loss": 3.1188, + "theoretical_loss": 4.10965199132619, + "tokens_seen": 323813376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045548645937813445, + "loss": 3.2839, + "theoretical_loss": 4.109555624159107, + "tokens_seen": 323878912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045547642928786357, + "loss": 3.3218, + "theoretical_loss": 4.109459281948317, + "tokens_seen": 323944448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554663991975928, + "loss": 2.8989, + "theoretical_loss": 4.109362964682312, + "tokens_seen": 324009984 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455456369107322, + "loss": 2.9077, + "theoretical_loss": 4.1092666723495865, + "tokens_seen": 324075520 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045544633901705117, + "loss": 3.2878, + "theoretical_loss": 4.109170404938649, + "tokens_seen": 324141056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045543630892678035, + "loss": 3.2166, + "theoretical_loss": 4.109074162438011, + "tokens_seen": 324206592 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045542627883650953, + "loss": 2.8978, + "theoretical_loss": 4.1089779448361945, + "tokens_seen": 324272128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554162487462387, + "loss": 3.1213, + "theoretical_loss": 4.108881752121727, + "tokens_seen": 324337664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 190150, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1001152992248535, + "objective/train/theoretical_loss": 4.108785584283144, + "objective/train/tokens_used": 344863200, + "theoretical_loss": 4.108785584283144, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045540621865596795, + "loss": 2.9022, + "theoretical_loss": 4.108785584283144, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553961885656971, + "loss": 2.9865, + "theoretical_loss": 4.108689441308989, + "tokens_seen": 324468736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553861584754263, + "loss": 3.112, + "theoretical_loss": 4.108593323187815, + "tokens_seen": 324534272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045537612838515544, + "loss": 3.1477, + "theoretical_loss": 4.108497229908179, + "tokens_seen": 324599808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553660982948847, + "loss": 3.0519, + "theoretical_loss": 4.108401161458646, + "tokens_seen": 324665344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045535606820461385, + "loss": 3.0665, + "theoretical_loss": 4.1083051178277925, + "tokens_seen": 324730880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045534603811434304, + "loss": 2.968, + "theoretical_loss": 4.108209099004198, + "tokens_seen": 324796416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553360080240722, + "loss": 3.1983, + "theoretical_loss": 4.108113104976453, + "tokens_seen": 324861952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045532597793380145, + "loss": 2.8336, + "theoretical_loss": 4.108017135733152, + "tokens_seen": 324927488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553159478435306, + "loss": 3.0941, + "theoretical_loss": 4.107921191262901, + "tokens_seen": 324993024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553059177532598, + "loss": 2.9691, + "theoretical_loss": 4.10782527155431, + "tokens_seen": 325058560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045529588766298894, + "loss": 2.9584, + "theoretical_loss": 4.107729376595999, + "tokens_seen": 325124096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552858575727182, + "loss": 3.253, + "theoretical_loss": 4.107633506376594, + "tokens_seen": 325189632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045527582748244736, + "loss": 3.0396, + "theoretical_loss": 4.10753766088473, + "tokens_seen": 325255168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045526579739217654, + "loss": 3.0754, + "theoretical_loss": 4.107441840109048, + "tokens_seen": 325320704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552557673019057, + "loss": 3.0014, + "theoretical_loss": 4.107346044038197, + "tokens_seen": 325386240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552457372116349, + "loss": 3.0876, + "theoretical_loss": 4.107250272660834, + "tokens_seen": 325451776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552357071213641, + "loss": 2.9613, + "theoretical_loss": 4.107154525965623, + "tokens_seen": 325517312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552256770310933, + "loss": 2.9588, + "theoretical_loss": 4.107058803941236, + "tokens_seen": 325582848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045521564694082244, + "loss": 2.9034, + "theoretical_loss": 4.106963106576352, + "tokens_seen": 325648384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552056168505517, + "loss": 2.8674, + "theoretical_loss": 4.106867433859656, + "tokens_seen": 325713920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551955867602808, + "loss": 2.9357, + "theoretical_loss": 4.106771785779845, + "tokens_seen": 325779456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045518555667001004, + "loss": 2.9561, + "theoretical_loss": 4.106676162325618, + "tokens_seen": 325844992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551755265797392, + "loss": 3.0155, + "theoretical_loss": 4.106580563485685, + "tokens_seen": 325910528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551654964894684, + "loss": 3.0204, + "theoretical_loss": 4.106484989248762, + "tokens_seen": 325976064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 191328, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0341267585754395, + "objective/train/theoretical_loss": 4.106389439603573, + "objective/train/tokens_used": 346501600, + "theoretical_loss": 4.106389439603573, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551554663991976, + "loss": 3.0402, + "theoretical_loss": 4.106389439603573, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551454363089268, + "loss": 3.1653, + "theoretical_loss": 4.10629391453885, + "tokens_seen": 326107136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045513540621865595, + "loss": 2.9886, + "theoretical_loss": 4.10619841404333, + "tokens_seen": 326172672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551253761283852, + "loss": 3.3446, + "theoretical_loss": 4.106102938105762, + "tokens_seen": 326238208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551153460381143, + "loss": 2.8962, + "theoretical_loss": 4.106007486714896, + "tokens_seen": 326303744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045510531594784354, + "loss": 3.1154, + "theoretical_loss": 4.105912059859495, + "tokens_seen": 326369280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550952858575727, + "loss": 3.048, + "theoretical_loss": 4.105816657528327, + "tokens_seen": 326434816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550852557673019, + "loss": 2.9685, + "theoretical_loss": 4.105721279710168, + "tokens_seen": 326500352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550752256770311, + "loss": 3.1754, + "theoretical_loss": 4.105625926393801, + "tokens_seen": 326565888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045506519558676027, + "loss": 3.2148, + "theoretical_loss": 4.105530597568015, + "tokens_seen": 326631424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045505516549648945, + "loss": 3.1651, + "theoretical_loss": 4.1054352932216105, + "tokens_seen": 326696960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550451354062187, + "loss": 3.0107, + "theoretical_loss": 4.10534001334339, + "tokens_seen": 326762496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550351053159478, + "loss": 3.2755, + "theoretical_loss": 4.105244757922169, + "tokens_seen": 326828032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045502507522567705, + "loss": 2.8913, + "theoretical_loss": 4.105149526946766, + "tokens_seen": 326893568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045501504513540623, + "loss": 2.9583, + "theoretical_loss": 4.1050543204060075, + "tokens_seen": 326959104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550050150451354, + "loss": 3.187, + "theoretical_loss": 4.10495913828873, + "tokens_seen": 327024640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549949849548646, + "loss": 2.9074, + "theoretical_loss": 4.104863980583774, + "tokens_seen": 327090176 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045498495486459377, + "loss": 3.144, + "theoretical_loss": 4.104768847279988, + "tokens_seen": 327155712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045497492477432295, + "loss": 3.0299, + "theoretical_loss": 4.104673738366232, + "tokens_seen": 327221248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549648946840522, + "loss": 2.9183, + "theoretical_loss": 4.104578653831368, + "tokens_seen": 327286784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045495486459378137, + "loss": 2.8461, + "theoretical_loss": 4.104483593664266, + "tokens_seen": 327352320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045494483450351055, + "loss": 2.9215, + "theoretical_loss": 4.104388557853807, + "tokens_seen": 327417856 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045493480441323973, + "loss": 3.0317, + "theoretical_loss": 4.104293546388876, + "tokens_seen": 327483392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549247743229689, + "loss": 3.0664, + "theoretical_loss": 4.104198559258364, + "tokens_seen": 327548928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045491474423269815, + "loss": 2.9383, + "theoretical_loss": 4.104103596451175, + "tokens_seen": 327614464 + }, + { + "debugging/Self-BLEU-5": 0.2643082570774211, + "debugging/distinct-1-grams": 0.8218199262737969, + "debugging/distinct-2-grams": 0.9963369963369964, + "debugging/entropy-1-grams": 4.917654182369999, + "debugging/entropy-2-grams": 5.417185152102512, + "debugging/length": 514.3333333333334, + "debugging/num_segments": 3, + "epoch": 0.1, + "objective/train/docs_used": 191945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.493633270263672, + "objective/train/theoretical_loss": 4.104008657956216, + "objective/train/tokens_used": 348140000, + "theoretical_loss": 4.104008657956216, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549047141424273, + "loss": 3.1165, + "theoretical_loss": 4.104008657956216, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548946840521565, + "loss": 2.9939, + "theoretical_loss": 4.1039137437624, + "tokens_seen": 327745536 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045488465396188564, + "loss": 2.8305, + "theoretical_loss": 4.103818853858651, + "tokens_seen": 327811072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548746238716149, + "loss": 3.0165, + "theoretical_loss": 4.103723988233898, + "tokens_seen": 327876608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045486459378134405, + "loss": 3.1865, + "theoretical_loss": 4.103629146877078, + "tokens_seen": 327942144 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045485456369107324, + "loss": 3.0442, + "theoretical_loss": 4.103534329777135, + "tokens_seen": 328007680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548445336008024, + "loss": 2.9175, + "theoretical_loss": 4.103439536923021, + "tokens_seen": 328073216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045483450351053165, + "loss": 2.9148, + "theoretical_loss": 4.103344768303694, + "tokens_seen": 328138752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548244734202608, + "loss": 3.2821, + "theoretical_loss": 4.103250023908119, + "tokens_seen": 328204288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045481444332999, + "loss": 3.0728, + "theoretical_loss": 4.103155303725271, + "tokens_seen": 328269824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045480441323971914, + "loss": 2.9522, + "theoretical_loss": 4.103060607744128, + "tokens_seen": 328335360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547943831494484, + "loss": 3.0337, + "theoretical_loss": 4.102965935953679, + "tokens_seen": 328400896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045478435305917756, + "loss": 2.7991, + "theoretical_loss": 4.102871288342918, + "tokens_seen": 328466432 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045477432296890674, + "loss": 2.9165, + "theoretical_loss": 4.102776664900848, + "tokens_seen": 328531968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547642928786359, + "loss": 3.0264, + "theoretical_loss": 4.1026820656164755, + "tokens_seen": 328597504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547542627883651, + "loss": 2.956, + "theoretical_loss": 4.102587490478819, + "tokens_seen": 328663040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547442326980943, + "loss": 3.0012, + "theoretical_loss": 4.102492939476901, + "tokens_seen": 328728576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547342026078235, + "loss": 3.2309, + "theoretical_loss": 4.102398412599753, + "tokens_seen": 328794112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045472417251755264, + "loss": 2.9335, + "theoretical_loss": 4.102303909836412, + "tokens_seen": 328859648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547141424272819, + "loss": 3.0824, + "theoretical_loss": 4.102209431175923, + "tokens_seen": 328925184 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454704112337011, + "loss": 3.0037, + "theoretical_loss": 4.102114976607338, + "tokens_seen": 328990720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045469408224674024, + "loss": 2.9785, + "theoretical_loss": 4.102020546119716, + "tokens_seen": 329056256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546840521564694, + "loss": 3.0101, + "theoretical_loss": 4.101926139702125, + "tokens_seen": 329121792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546740220661986, + "loss": 3.1876, + "theoretical_loss": 4.101831757343637, + "tokens_seen": 329187328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546639919759278, + "loss": 3.0554, + "theoretical_loss": 4.1017373990333335, + "tokens_seen": 329252864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 193205, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.53017520904541, + "objective/train/theoretical_loss": 4.101643064760301, + "objective/train/tokens_used": 349778400, + "theoretical_loss": 4.101643064760301, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454653961885657, + "loss": 2.8743, + "theoretical_loss": 4.101643064760301, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045464393179538615, + "loss": 2.7826, + "theoretical_loss": 4.101548754513636, + "tokens_seen": 329383936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546339017051154, + "loss": 2.8613, + "theoretical_loss": 4.101454468282439, + "tokens_seen": 329449472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546238716148445, + "loss": 2.9591, + "theoretical_loss": 4.101360206055821, + "tokens_seen": 329515008 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045461384152457374, + "loss": 3.0603, + "theoretical_loss": 4.101265967822897, + "tokens_seen": 329580544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546038114343029, + "loss": 2.8043, + "theoretical_loss": 4.101171753572791, + "tokens_seen": 329646080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545937813440321, + "loss": 3.0562, + "theoretical_loss": 4.101077563294632, + "tokens_seen": 329711616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545837512537613, + "loss": 3.1101, + "theoretical_loss": 4.1009833969775595, + "tokens_seen": 329777152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045457372116349047, + "loss": 3.162, + "theoretical_loss": 4.100889254610717, + "tokens_seen": 329842688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045456369107321965, + "loss": 2.9801, + "theoretical_loss": 4.100795136183256, + "tokens_seen": 329908224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545536609829489, + "loss": 2.9315, + "theoretical_loss": 4.100701041684336, + "tokens_seen": 329973760 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454543630892678, + "loss": 3.097, + "theoretical_loss": 4.100606971103122, + "tokens_seen": 330039296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045453360080240725, + "loss": 3.0641, + "theoretical_loss": 4.100512924428788, + "tokens_seen": 330104832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045452357071213643, + "loss": 2.9796, + "theoretical_loss": 4.100418901650513, + "tokens_seen": 330170368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545135406218656, + "loss": 2.9823, + "theoretical_loss": 4.1003249027574835, + "tokens_seen": 330235904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545035105315948, + "loss": 3.0358, + "theoretical_loss": 4.100230927738895, + "tokens_seen": 330301440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045449348044132397, + "loss": 3.0782, + "theoretical_loss": 4.100136976583947, + "tokens_seen": 330366976 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045448345035105315, + "loss": 3.0677, + "theoretical_loss": 4.100043049281849, + "tokens_seen": 330432512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544734202607824, + "loss": 2.9041, + "theoretical_loss": 4.099949145821815, + "tokens_seen": 330498048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544633901705115, + "loss": 3.1047, + "theoretical_loss": 4.099855266193067, + "tokens_seen": 330563584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045445336008024075, + "loss": 3.2029, + "theoretical_loss": 4.099761410384835, + "tokens_seen": 330629120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544433299899699, + "loss": 2.8177, + "theoretical_loss": 4.099667578386355, + "tokens_seen": 330694656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544332998996991, + "loss": 2.8579, + "theoretical_loss": 4.099573770186869, + "tokens_seen": 330760192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544232698094283, + "loss": 2.9416, + "theoretical_loss": 4.099479985775629, + "tokens_seen": 330825728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544132397191575, + "loss": 2.997, + "theoretical_loss": 4.09938622514189, + "tokens_seen": 330891264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 193823, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6776926517486572, + "objective/train/theoretical_loss": 4.099292488274917, + "objective/train/tokens_used": 351416800, + "theoretical_loss": 4.099292488274917, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045440320962888666, + "loss": 2.9025, + "theoretical_loss": 4.099292488274917, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045439317953861584, + "loss": 2.9272, + "theoretical_loss": 4.0991987751639805, + "tokens_seen": 331022336 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454383149448345, + "loss": 3.0293, + "theoretical_loss": 4.099105085798359, + "tokens_seen": 331087872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045437311935807425, + "loss": 2.9882, + "theoretical_loss": 4.099011420167338, + "tokens_seen": 331153408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543630892678034, + "loss": 2.9665, + "theoretical_loss": 4.098917778260208, + "tokens_seen": 331218944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543530591775326, + "loss": 3.145, + "theoretical_loss": 4.098824160066269, + "tokens_seen": 331284480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543430290872618, + "loss": 3.0804, + "theoretical_loss": 4.098730565574826, + "tokens_seen": 331350016 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454332998996991, + "loss": 2.8699, + "theoretical_loss": 4.098636994775193, + "tokens_seen": 331415552 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045432296890672016, + "loss": 2.8012, + "theoretical_loss": 4.098543447656688, + "tokens_seen": 331481088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045431293881644934, + "loss": 3.1938, + "theoretical_loss": 4.098449924208639, + "tokens_seen": 331546624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543029087261785, + "loss": 2.8317, + "theoretical_loss": 4.098356424420379, + "tokens_seen": 331612160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045429287863590776, + "loss": 2.9114, + "theoretical_loss": 4.098262948281247, + "tokens_seen": 331677696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542828485456369, + "loss": 2.9306, + "theoretical_loss": 4.098169495780592, + "tokens_seen": 331743232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542728184553661, + "loss": 3.0543, + "theoretical_loss": 4.098076066907769, + "tokens_seen": 331808768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045426278836509525, + "loss": 2.9948, + "theoretical_loss": 4.0979826616521375, + "tokens_seen": 331874304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542527582748245, + "loss": 3.0341, + "theoretical_loss": 4.0978892800030655, + "tokens_seen": 331939840 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045424272818455366, + "loss": 2.7333, + "theoretical_loss": 4.0977959219499285, + "tokens_seen": 332005376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045423269809428284, + "loss": 2.7174, + "theoretical_loss": 4.097702587482107, + "tokens_seen": 332070912 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454222668004012, + "loss": 2.9369, + "theoretical_loss": 4.097609276588993, + "tokens_seen": 332136448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542126379137412, + "loss": 2.9834, + "theoretical_loss": 4.097515989259978, + "tokens_seen": 332201984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045420260782347044, + "loss": 2.7845, + "theoretical_loss": 4.097422725484467, + "tokens_seen": 332267520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541925777331996, + "loss": 2.7439, + "theoretical_loss": 4.097329485251869, + "tokens_seen": 332333056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541825476429288, + "loss": 3.1478, + "theoretical_loss": 4.097236268551598, + "tokens_seen": 332398592 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454172517552658, + "loss": 2.7357, + "theoretical_loss": 4.097143075373079, + "tokens_seen": 332464128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541624874623872, + "loss": 2.9623, + "theoretical_loss": 4.097049905705742, + "tokens_seen": 332529664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 194956, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.295325756072998, + "objective/train/theoretical_loss": 4.0969567595390215, + "objective/train/tokens_used": 353055200, + "theoretical_loss": 4.0969567595390215, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045415245737211635, + "loss": 3.0697, + "theoretical_loss": 4.0969567595390215, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541424272818456, + "loss": 2.9866, + "theoretical_loss": 4.096863636862363, + "tokens_seen": 332660736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541323971915747, + "loss": 3.0612, + "theoretical_loss": 4.096770537665216, + "tokens_seen": 332726272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045412236710130395, + "loss": 2.8517, + "theoretical_loss": 4.096677461937037, + "tokens_seen": 332791808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541123370110331, + "loss": 3.2886, + "theoretical_loss": 4.09658440966729, + "tokens_seen": 332857344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541023069207623, + "loss": 3.0414, + "theoretical_loss": 4.096491380845446, + "tokens_seen": 332922880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540922768304915, + "loss": 3.1045, + "theoretical_loss": 4.096398375460983, + "tokens_seen": 332988416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045408224674022067, + "loss": 3.19, + "theoretical_loss": 4.096305393503385, + "tokens_seen": 333053952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045407221664994985, + "loss": 2.9571, + "theoretical_loss": 4.096212434962142, + "tokens_seen": 333119488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540621865596791, + "loss": 2.8609, + "theoretical_loss": 4.096119499826752, + "tokens_seen": 333185024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540521564694082, + "loss": 3.079, + "theoretical_loss": 4.096026588086722, + "tokens_seen": 333250560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045404212637913745, + "loss": 2.9375, + "theoretical_loss": 4.09593369973156, + "tokens_seen": 333316096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045403209628886663, + "loss": 3.0762, + "theoretical_loss": 4.095840834750787, + "tokens_seen": 333381632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540220661985958, + "loss": 3.0485, + "theoretical_loss": 4.095747993133925, + "tokens_seen": 333447168 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454012036108325, + "loss": 2.8056, + "theoretical_loss": 4.095655174870508, + "tokens_seen": 333512704 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045400200601805417, + "loss": 2.9316, + "theoretical_loss": 4.095562379950073, + "tokens_seen": 333578240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045399197592778335, + "loss": 3.156, + "theoretical_loss": 4.0954696083621664, + "tokens_seen": 333643776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539819458375126, + "loss": 2.8985, + "theoretical_loss": 4.095376860096339, + "tokens_seen": 333709312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539719157472417, + "loss": 2.944, + "theoretical_loss": 4.095284135142149, + "tokens_seen": 333774848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045396188565697095, + "loss": 2.8726, + "theoretical_loss": 4.095191433489164, + "tokens_seen": 333840384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539518555667001, + "loss": 2.8167, + "theoretical_loss": 4.095098755126954, + "tokens_seen": 333905920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539418254764293, + "loss": 2.859, + "theoretical_loss": 4.095006100045097, + "tokens_seen": 333971456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539317953861585, + "loss": 2.9604, + "theoretical_loss": 4.094913468233182, + "tokens_seen": 334036992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539217652958877, + "loss": 2.9834, + "theoretical_loss": 4.094820859680797, + "tokens_seen": 334102528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045391173520561686, + "loss": 2.9309, + "theoretical_loss": 4.094728274377543, + "tokens_seen": 334168064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 196277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.767504930496216, + "objective/train/theoretical_loss": 4.094635712313026, + "objective/train/tokens_used": 354693600, + "theoretical_loss": 4.094635712313026, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045390170511534604, + "loss": 3.0447, + "theoretical_loss": 4.094635712313026, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538916750250752, + "loss": 3.0314, + "theoretical_loss": 4.094543173476857, + "tokens_seen": 334299136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045388164493480445, + "loss": 3.1375, + "theoretical_loss": 4.094450657858655, + "tokens_seen": 334364672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538716148445336, + "loss": 2.9004, + "theoretical_loss": 4.094358165448046, + "tokens_seen": 334430208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538615847542628, + "loss": 2.936, + "theoretical_loss": 4.094265696234663, + "tokens_seen": 334495744 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453851554663992, + "loss": 2.9983, + "theoretical_loss": 4.094173250208143, + "tokens_seen": 334561280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538415245737212, + "loss": 3.0602, + "theoretical_loss": 4.094080827358133, + "tokens_seen": 334626816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045383149448345036, + "loss": 2.8065, + "theoretical_loss": 4.093988427674285, + "tokens_seen": 334692352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045382146439317954, + "loss": 2.8173, + "theoretical_loss": 4.093896051146257, + "tokens_seen": 334757888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538114343029087, + "loss": 2.9987, + "theoretical_loss": 4.0938036977637156, + "tokens_seen": 334823424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045380140421263796, + "loss": 2.9723, + "theoretical_loss": 4.093711367516333, + "tokens_seen": 334888960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537913741223671, + "loss": 3.1855, + "theoretical_loss": 4.093619060393786, + "tokens_seen": 334954496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537813440320963, + "loss": 3.0542, + "theoretical_loss": 4.0935267763857635, + "tokens_seen": 335020032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045377131394182545, + "loss": 3.0719, + "theoretical_loss": 4.093434515481954, + "tokens_seen": 335085568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537612838515547, + "loss": 2.7748, + "theoretical_loss": 4.093342277672058, + "tokens_seen": 335151104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045375125376128386, + "loss": 2.9491, + "theoretical_loss": 4.09325006294578, + "tokens_seen": 335216640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045374122367101304, + "loss": 2.9779, + "theoretical_loss": 4.093157871292833, + "tokens_seen": 335282176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537311935807422, + "loss": 2.8597, + "theoretical_loss": 4.093065702702933, + "tokens_seen": 335347712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537211634904714, + "loss": 3.029, + "theoretical_loss": 4.0929735571658075, + "tokens_seen": 335413248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537111334002006, + "loss": 3.136, + "theoretical_loss": 4.092881434671186, + "tokens_seen": 335478784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537011033099298, + "loss": 3.1326, + "theoretical_loss": 4.09278933520881, + "tokens_seen": 335544320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045369107321965895, + "loss": 2.7854, + "theoretical_loss": 4.092697258768419, + "tokens_seen": 335609856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536810431293882, + "loss": 3.006, + "theoretical_loss": 4.09260520533977, + "tokens_seen": 335675392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045367101303911737, + "loss": 3.2209, + "theoretical_loss": 4.092513174912616, + "tokens_seen": 335740928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045366098294884655, + "loss": 3.0421, + "theoretical_loss": 4.092421167476726, + "tokens_seen": 335806464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 196731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1563823223114014, + "objective/train/theoretical_loss": 4.0923291830218655, + "objective/train/tokens_used": 356332000, + "theoretical_loss": 4.0923291830218655, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045365095285857573, + "loss": 3.0343, + "theoretical_loss": 4.0923291830218655, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536409227683049, + "loss": 3.1099, + "theoretical_loss": 4.092237221537817, + "tokens_seen": 335937536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536308926780341, + "loss": 3.1611, + "theoretical_loss": 4.0921452830143625, + "tokens_seen": 336003072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536208625877633, + "loss": 3.0936, + "theoretical_loss": 4.092053367441292, + "tokens_seen": 336068608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045361083249749245, + "loss": 3.0695, + "theoretical_loss": 4.091961474808404, + "tokens_seen": 336134144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536008024072217, + "loss": 3.1083, + "theoretical_loss": 4.091869605105501, + "tokens_seen": 336199680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535907723169508, + "loss": 2.9725, + "theoretical_loss": 4.091777758322394, + "tokens_seen": 336265216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045358074222668005, + "loss": 2.7863, + "theoretical_loss": 4.091685934448899, + "tokens_seen": 336330752 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045357071213640923, + "loss": 2.7613, + "theoretical_loss": 4.09159413347484, + "tokens_seen": 336396288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535606820461384, + "loss": 2.7013, + "theoretical_loss": 4.091502355390046, + "tokens_seen": 336461824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535506519558676, + "loss": 3.0298, + "theoretical_loss": 4.091410600184353, + "tokens_seen": 336527360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045354062186559683, + "loss": 2.8801, + "theoretical_loss": 4.091318867847605, + "tokens_seen": 336592896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045353059177532596, + "loss": 2.9488, + "theoretical_loss": 4.091227158369649, + "tokens_seen": 336658432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535205616850552, + "loss": 2.8764, + "theoretical_loss": 4.091135471740342, + "tokens_seen": 336723968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535105315947843, + "loss": 2.994, + "theoretical_loss": 4.091043807949546, + "tokens_seen": 336789504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045350050150451355, + "loss": 2.947, + "theoretical_loss": 4.0909521669871305, + "tokens_seen": 336855040 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045349047141424274, + "loss": 3.1228, + "theoretical_loss": 4.0908605488429695, + "tokens_seen": 336920576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534804413239719, + "loss": 2.939, + "theoretical_loss": 4.090768953506944, + "tokens_seen": 336986112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534704112337011, + "loss": 2.8694, + "theoretical_loss": 4.090677380968943, + "tokens_seen": 337051648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534603811434303, + "loss": 2.9003, + "theoretical_loss": 4.09058583121886, + "tokens_seen": 337117184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534503510531595, + "loss": 2.9993, + "theoretical_loss": 4.090494304246597, + "tokens_seen": 337182720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534403209628887, + "loss": 3.1168, + "theoretical_loss": 4.09040280004206, + "tokens_seen": 337248256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534302908726179, + "loss": 2.9275, + "theoretical_loss": 4.090311318595164, + "tokens_seen": 337313792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045342026078234706, + "loss": 2.941, + "theoretical_loss": 4.090219859895829, + "tokens_seen": 337379328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045341023069207624, + "loss": 2.8286, + "theoretical_loss": 4.09012842393398, + "tokens_seen": 337444864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 198039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.190596342086792, + "objective/train/theoretical_loss": 4.090037010699552, + "objective/train/tokens_used": 357970400, + "theoretical_loss": 4.090037010699552, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534002006018054, + "loss": 3.0192, + "theoretical_loss": 4.090037010699552, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045339017051153465, + "loss": 2.921, + "theoretical_loss": 4.089945620182483, + "tokens_seen": 337575936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533801404212638, + "loss": 3.1205, + "theoretical_loss": 4.08985425237272, + "tokens_seen": 337641472 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453370110330993, + "loss": 2.8657, + "theoretical_loss": 4.089762907260214, + "tokens_seen": 337707008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533600802407222, + "loss": 2.9334, + "theoretical_loss": 4.0896715848349245, + "tokens_seen": 337772544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533500501504514, + "loss": 2.8687, + "theoretical_loss": 4.089580285086815, + "tokens_seen": 337838080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045334002006018056, + "loss": 2.9244, + "theoretical_loss": 4.0894890080058595, + "tokens_seen": 337903616 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045332998996990974, + "loss": 2.7939, + "theoretical_loss": 4.089397753582034, + "tokens_seen": 337969152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533199598796389, + "loss": 2.9034, + "theoretical_loss": 4.089306521805322, + "tokens_seen": 338034688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045330992978936816, + "loss": 2.8053, + "theoretical_loss": 4.089215312665716, + "tokens_seen": 338100224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532998996990973, + "loss": 3.0222, + "theoretical_loss": 4.0891241261532105, + "tokens_seen": 338165760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532898696088265, + "loss": 3.2622, + "theoretical_loss": 4.089032962257811, + "tokens_seen": 338231296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045327983951855565, + "loss": 3.0372, + "theoretical_loss": 4.088941820969525, + "tokens_seen": 338296832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532698094282849, + "loss": 2.8771, + "theoretical_loss": 4.08885070227837, + "tokens_seen": 338362368 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045325977933801406, + "loss": 2.9213, + "theoretical_loss": 4.088759606174369, + "tokens_seen": 338427904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045324974924774324, + "loss": 2.877, + "theoretical_loss": 4.088668532647548, + "tokens_seen": 338493440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532397191574724, + "loss": 2.9616, + "theoretical_loss": 4.088577481687944, + "tokens_seen": 338558976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532296890672016, + "loss": 2.9701, + "theoretical_loss": 4.088486453285596, + "tokens_seen": 338624512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532196589769308, + "loss": 2.9765, + "theoretical_loss": 4.088395447430554, + "tokens_seen": 338690048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045320962888666, + "loss": 2.9125, + "theoretical_loss": 4.088304464112872, + "tokens_seen": 338755584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045319959879638915, + "loss": 2.7839, + "theoretical_loss": 4.088213503322607, + "tokens_seen": 338821120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531895687061184, + "loss": 2.8615, + "theoretical_loss": 4.088122565049829, + "tokens_seen": 338886656 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045317953861584757, + "loss": 3.0402, + "theoretical_loss": 4.088031649284609, + "tokens_seen": 338952192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045316950852557675, + "loss": 3.1794, + "theoretical_loss": 4.087940756017027, + "tokens_seen": 339017728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045315947843530593, + "loss": 2.8781, + "theoretical_loss": 4.087849885237168, + "tokens_seen": 339083264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 198530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.366729259490967, + "objective/train/theoretical_loss": 4.0877590369351235, + "objective/train/tokens_used": 359608800, + "theoretical_loss": 4.0877590369351235, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531494483450351, + "loss": 3.1109, + "theoretical_loss": 4.0877590369351235, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531394182547643, + "loss": 2.8525, + "theoretical_loss": 4.0876682111009925, + "tokens_seen": 339214336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531293881644935, + "loss": 3.1438, + "theoretical_loss": 4.087577407724877, + "tokens_seen": 339279872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045311935807422265, + "loss": 2.836, + "theoretical_loss": 4.08748662679689, + "tokens_seen": 339345408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531093279839519, + "loss": 2.8101, + "theoretical_loss": 4.087395868307146, + "tokens_seen": 339410944 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453099297893681, + "loss": 3.1357, + "theoretical_loss": 4.08730513224577, + "tokens_seen": 339476480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045308926780341025, + "loss": 3.14, + "theoretical_loss": 4.08721441860289, + "tokens_seen": 339542016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045307923771313943, + "loss": 3.0339, + "theoretical_loss": 4.087123727368642, + "tokens_seen": 339607552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530692076228686, + "loss": 3.1427, + "theoretical_loss": 4.087033058533167, + "tokens_seen": 339673088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530591775325978, + "loss": 3.0171, + "theoretical_loss": 4.086942412086614, + "tokens_seen": 339738624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045304914744232703, + "loss": 3.0101, + "theoretical_loss": 4.086851788019137, + "tokens_seen": 339804160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045303911735205616, + "loss": 3.0004, + "theoretical_loss": 4.086761186320897, + "tokens_seen": 339869696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530290872617854, + "loss": 3.0109, + "theoretical_loss": 4.086670606982059, + "tokens_seen": 339935232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530190571715145, + "loss": 2.7429, + "theoretical_loss": 4.086580049992798, + "tokens_seen": 340000768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045300902708124375, + "loss": 2.9896, + "theoretical_loss": 4.086489515343292, + "tokens_seen": 340066304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045299899699097294, + "loss": 3.0274, + "theoretical_loss": 4.086399003023725, + "tokens_seen": 340131840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529889669007021, + "loss": 3.1022, + "theoretical_loss": 4.086308513024291, + "tokens_seen": 340197376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529789368104313, + "loss": 2.9777, + "theoretical_loss": 4.086218045335187, + "tokens_seen": 340262912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529689067201605, + "loss": 3.1257, + "theoretical_loss": 4.086127599946616, + "tokens_seen": 340328448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045295887662988966, + "loss": 3.0384, + "theoretical_loss": 4.086037176848789, + "tokens_seen": 340393984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529488465396189, + "loss": 3.0784, + "theoretical_loss": 4.085946776031922, + "tokens_seen": 340459520 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452938816449348, + "loss": 2.9653, + "theoretical_loss": 4.085856397486237, + "tokens_seen": 340525056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045292878635907726, + "loss": 2.9517, + "theoretical_loss": 4.085766041201964, + "tokens_seen": 340590592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529187562688064, + "loss": 2.742, + "theoretical_loss": 4.085675707169336, + "tokens_seen": 340656128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529087261785356, + "loss": 2.8849, + "theoretical_loss": 4.085585395378595, + "tokens_seen": 340721664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 199795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8782248497009277, + "objective/train/theoretical_loss": 4.085495105819987, + "objective/train/tokens_used": 361247200, + "theoretical_loss": 4.085495105819987, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528986960882648, + "loss": 2.8611, + "theoretical_loss": 4.085495105819987, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452888665997994, + "loss": 3.0898, + "theoretical_loss": 4.085404838483767, + "tokens_seen": 340852736 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045287863590772316, + "loss": 2.9958, + "theoretical_loss": 4.085314593360194, + "tokens_seen": 340918272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528686058174524, + "loss": 3.0597, + "theoretical_loss": 4.085224370439533, + "tokens_seen": 340983808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528585757271815, + "loss": 2.7746, + "theoretical_loss": 4.085134169712055, + "tokens_seen": 341049344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045284854563691076, + "loss": 2.9852, + "theoretical_loss": 4.085043991168039, + "tokens_seen": 341114880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528385155466399, + "loss": 3.0912, + "theoretical_loss": 4.084953834797769, + "tokens_seen": 341180416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528284854563691, + "loss": 2.96, + "theoretical_loss": 4.0848637005915345, + "tokens_seen": 341245952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528184553660983, + "loss": 3.2266, + "theoretical_loss": 4.0847735885396315, + "tokens_seen": 341311488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528084252758275, + "loss": 2.9558, + "theoretical_loss": 4.084683498632363, + "tokens_seen": 341377024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045279839518555667, + "loss": 3.0659, + "theoretical_loss": 4.084593430860037, + "tokens_seen": 341442560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045278836509528585, + "loss": 2.8982, + "theoretical_loss": 4.084503385212968, + "tokens_seen": 341508096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045277833500501503, + "loss": 2.7515, + "theoretical_loss": 4.084413361681476, + "tokens_seen": 341573632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045276830491474426, + "loss": 3.1749, + "theoretical_loss": 4.084323360255889, + "tokens_seen": 341639168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527582748244734, + "loss": 2.8382, + "theoretical_loss": 4.0842333809265385, + "tokens_seen": 341704704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527482447342026, + "loss": 2.8088, + "theoretical_loss": 4.084143423683765, + "tokens_seen": 341770240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045273821464393175, + "loss": 2.8347, + "theoretical_loss": 4.0840534885179105, + "tokens_seen": 341835776 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452728184553661, + "loss": 3.0138, + "theoretical_loss": 4.083963575419329, + "tokens_seen": 341901312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045271815446339017, + "loss": 2.9717, + "theoretical_loss": 4.083873684378376, + "tokens_seen": 341966848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045270812437311935, + "loss": 2.9964, + "theoretical_loss": 4.083783815385415, + "tokens_seen": 342032384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526980942828486, + "loss": 3.0922, + "theoretical_loss": 4.083693968430816, + "tokens_seen": 342097920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045268806419257777, + "loss": 2.9981, + "theoretical_loss": 4.083604143504952, + "tokens_seen": 342163456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045267803410230695, + "loss": 3.4607, + "theoretical_loss": 4.083514340598207, + "tokens_seen": 342228992 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045266800401203613, + "loss": 3.0061, + "theoretical_loss": 4.083424559700966, + "tokens_seen": 342294528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526579739217653, + "loss": 2.9478, + "theoretical_loss": 4.083334800803624, + "tokens_seen": 342360064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 200497, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0360288619995117, + "objective/train/theoretical_loss": 4.08324506389658, + "objective/train/tokens_used": 362885600, + "theoretical_loss": 4.08324506389658, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526479438314945, + "loss": 2.9394, + "theoretical_loss": 4.08324506389658, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526379137412237, + "loss": 2.9863, + "theoretical_loss": 4.083155348970238, + "tokens_seen": 342491136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045262788365095285, + "loss": 2.9571, + "theoretical_loss": 4.083065656015012, + "tokens_seen": 342556672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526178535606821, + "loss": 3.2024, + "theoretical_loss": 4.082975985021317, + "tokens_seen": 342622208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526078234704112, + "loss": 3.1355, + "theoretical_loss": 4.082886335979578, + "tokens_seen": 342687744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045259779338014045, + "loss": 2.9816, + "theoretical_loss": 4.082796708880224, + "tokens_seen": 342753280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045258776328986963, + "loss": 3.1751, + "theoretical_loss": 4.082707103713689, + "tokens_seen": 342818816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525777331995988, + "loss": 2.8529, + "theoretical_loss": 4.082617520470418, + "tokens_seen": 342884352 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452567703109328, + "loss": 3.0098, + "theoretical_loss": 4.0825279591408545, + "tokens_seen": 342949888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045255767301905723, + "loss": 2.8312, + "theoretical_loss": 4.082438419715455, + "tokens_seen": 343015424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045254764292878636, + "loss": 2.8357, + "theoretical_loss": 4.082348902184677, + "tokens_seen": 343080960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525376128385156, + "loss": 2.9867, + "theoretical_loss": 4.082259406538987, + "tokens_seen": 343146496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525275827482447, + "loss": 2.9535, + "theoretical_loss": 4.082169932768856, + "tokens_seen": 343212032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045251755265797395, + "loss": 2.8995, + "theoretical_loss": 4.082080480864761, + "tokens_seen": 343277568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045250752256770314, + "loss": 3.0748, + "theoretical_loss": 4.0819910508171855, + "tokens_seen": 343343104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524974924774323, + "loss": 2.9816, + "theoretical_loss": 4.08190164261662, + "tokens_seen": 343408640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524874623871615, + "loss": 2.9615, + "theoretical_loss": 4.081812256253557, + "tokens_seen": 343474176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524774322968907, + "loss": 3.0754, + "theoretical_loss": 4.0817228917185, + "tokens_seen": 343539712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045246740220661986, + "loss": 3.277, + "theoretical_loss": 4.081633549001955, + "tokens_seen": 343605248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524573721163491, + "loss": 2.9695, + "theoretical_loss": 4.081544228094436, + "tokens_seen": 343670784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524473420260782, + "loss": 3.0677, + "theoretical_loss": 4.08145492898646, + "tokens_seen": 343736320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045243731193580746, + "loss": 3.0617, + "theoretical_loss": 4.081365651668555, + "tokens_seen": 343801856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524272818455366, + "loss": 2.8337, + "theoretical_loss": 4.081276396131249, + "tokens_seen": 343867392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524172517552658, + "loss": 2.9259, + "theoretical_loss": 4.0811871623650795, + "tokens_seen": 343932928 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452407221664995, + "loss": 3.2243, + "theoretical_loss": 4.08109795036059, + "tokens_seen": 343998464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 201708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.375955820083618, + "objective/train/theoretical_loss": 4.081008760108329, + "objective/train/tokens_used": 364524000, + "theoretical_loss": 4.081008760108329, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523971915747242, + "loss": 3.0911, + "theoretical_loss": 4.081008760108329, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045238716148445336, + "loss": 2.8323, + "theoretical_loss": 4.08091959159885, + "tokens_seen": 344129536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523771313941826, + "loss": 3.0438, + "theoretical_loss": 4.0808304448227135, + "tokens_seen": 344195072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523671013039117, + "loss": 3.0118, + "theoretical_loss": 4.080741319770486, + "tokens_seen": 344260608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045235707121364096, + "loss": 3.1077, + "theoretical_loss": 4.080652216432741, + "tokens_seen": 344326144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523470411233701, + "loss": 2.9864, + "theoretical_loss": 4.0805631348000535, + "tokens_seen": 344391680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523370110330993, + "loss": 2.7933, + "theoretical_loss": 4.080474074863011, + "tokens_seen": 344457216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523269809428285, + "loss": 3.0455, + "theoretical_loss": 4.0803850366122, + "tokens_seen": 344522752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523169508525577, + "loss": 3.0973, + "theoretical_loss": 4.080296020038219, + "tokens_seen": 344588288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045230692076228687, + "loss": 2.8627, + "theoretical_loss": 4.080207025131667, + "tokens_seen": 344653824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045229689067201605, + "loss": 2.7906, + "theoretical_loss": 4.0801180518831535, + "tokens_seen": 344719360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045228686058174523, + "loss": 2.7051, + "theoretical_loss": 4.080029100283291, + "tokens_seen": 344784896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045227683049147446, + "loss": 3.1299, + "theoretical_loss": 4.079940170322698, + "tokens_seen": 344850432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522668004012036, + "loss": 2.7122, + "theoretical_loss": 4.079851261992, + "tokens_seen": 344915968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522567703109328, + "loss": 2.7792, + "theoretical_loss": 4.0797623752818275, + "tokens_seen": 344981504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045224674022066195, + "loss": 3.1825, + "theoretical_loss": 4.079673510182818, + "tokens_seen": 345047040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522367101303912, + "loss": 2.8207, + "theoretical_loss": 4.079584666685612, + "tokens_seen": 345112576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045222668004012037, + "loss": 2.9957, + "theoretical_loss": 4.07949584478086, + "tokens_seen": 345178112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045221664994984955, + "loss": 2.8706, + "theoretical_loss": 4.079407044459216, + "tokens_seen": 345243648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045220661985957873, + "loss": 2.7394, + "theoretical_loss": 4.079318265711338, + "tokens_seen": 345309184 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045219658976930797, + "loss": 2.7548, + "theoretical_loss": 4.0792295085278925, + "tokens_seen": 345374720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521865596790371, + "loss": 2.9065, + "theoretical_loss": 4.079140772899552, + "tokens_seen": 345440256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045217652958876633, + "loss": 3.0047, + "theoretical_loss": 4.079052058816993, + "tokens_seen": 345505792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045216649949849546, + "loss": 3.1107, + "theoretical_loss": 4.078963366270899, + "tokens_seen": 345571328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521564694082247, + "loss": 2.7583, + "theoretical_loss": 4.078874695251959, + "tokens_seen": 345636864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 202216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1073813438415527, + "objective/train/theoretical_loss": 4.078786045750867, + "objective/train/tokens_used": 366162400, + "theoretical_loss": 4.078786045750867, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045214643931795387, + "loss": 2.8222, + "theoretical_loss": 4.078786045750867, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045213640922768305, + "loss": 3.0401, + "theoretical_loss": 4.078697417758325, + "tokens_seen": 345767936 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045212637913741223, + "loss": 3.0623, + "theoretical_loss": 4.078608811265037, + "tokens_seen": 345833472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521163490471414, + "loss": 2.8066, + "theoretical_loss": 4.078520226261718, + "tokens_seen": 345899008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521063189568706, + "loss": 2.8332, + "theoretical_loss": 4.078431662739083, + "tokens_seen": 345964544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045209628886659983, + "loss": 3.0442, + "theoretical_loss": 4.078343120687858, + "tokens_seen": 346030080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045208625877632896, + "loss": 3.0793, + "theoretical_loss": 4.078254600098771, + "tokens_seen": 346095616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520762286860582, + "loss": 3.2073, + "theoretical_loss": 4.078166100962558, + "tokens_seen": 346161152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520661985957873, + "loss": 2.9417, + "theoretical_loss": 4.078077623269959, + "tokens_seen": 346226688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045205616850551656, + "loss": 2.9708, + "theoretical_loss": 4.0779891670117205, + "tokens_seen": 346292224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045204613841524574, + "loss": 3.0795, + "theoretical_loss": 4.077900732178595, + "tokens_seen": 346357760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520361083249749, + "loss": 3.1212, + "theoretical_loss": 4.0778123187613415, + "tokens_seen": 346423296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520260782347041, + "loss": 3.0503, + "theoretical_loss": 4.077723926750724, + "tokens_seen": 346488832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045201604814443334, + "loss": 2.8794, + "theoretical_loss": 4.07763555613751, + "tokens_seen": 346554368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045200601805416246, + "loss": 3.0506, + "theoretical_loss": 4.077547206912477, + "tokens_seen": 346619904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519959879638917, + "loss": 3.0215, + "theoretical_loss": 4.077458879066406, + "tokens_seen": 346685440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519859578736208, + "loss": 2.9088, + "theoretical_loss": 4.077370572590082, + "tokens_seen": 346750976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045197592778335006, + "loss": 2.8665, + "theoretical_loss": 4.077282287474299, + "tokens_seen": 346816512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045196589769307924, + "loss": 3.084, + "theoretical_loss": 4.077194023709854, + "tokens_seen": 346882048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519558676028084, + "loss": 2.9599, + "theoretical_loss": 4.077105781287552, + "tokens_seen": 346947584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045194583751253766, + "loss": 2.9752, + "theoretical_loss": 4.077017560198202, + "tokens_seen": 347013120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519358074222668, + "loss": 3.182, + "theoretical_loss": 4.0769293604326196, + "tokens_seen": 347078656 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451925777331996, + "loss": 2.9303, + "theoretical_loss": 4.076841181981624, + "tokens_seen": 347144192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519157472417252, + "loss": 3.0146, + "theoretical_loss": 4.076753024836044, + "tokens_seen": 347209728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519057171514544, + "loss": 3.2817, + "theoretical_loss": 4.076664888986713, + "tokens_seen": 347275264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 203448, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4970381259918213, + "objective/train/theoretical_loss": 4.076576774424465, + "objective/train/tokens_used": 367800800, + "theoretical_loss": 4.076576774424465, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045189568706118356, + "loss": 2.8287, + "theoretical_loss": 4.076576774424465, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518856569709128, + "loss": 2.8508, + "theoretical_loss": 4.0764886811401455, + "tokens_seen": 347406336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518756268806419, + "loss": 2.837, + "theoretical_loss": 4.0764006091246054, + "tokens_seen": 347471872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045186559679037116, + "loss": 3.1222, + "theoretical_loss": 4.076312558368698, + "tokens_seen": 347537408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518555667001003, + "loss": 2.8597, + "theoretical_loss": 4.076224528863284, + "tokens_seen": 347602944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518455366098295, + "loss": 2.9747, + "theoretical_loss": 4.0761365205992295, + "tokens_seen": 347668480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518355065195587, + "loss": 2.8766, + "theoretical_loss": 4.076048533567406, + "tokens_seen": 347734016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518254764292879, + "loss": 3.0041, + "theoretical_loss": 4.075960567758694, + "tokens_seen": 347799552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045181544633901707, + "loss": 2.9555, + "theoretical_loss": 4.075872623163973, + "tokens_seen": 347865088 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045180541624874625, + "loss": 2.7585, + "theoretical_loss": 4.075784699774134, + "tokens_seen": 347930624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045179538615847543, + "loss": 2.9776, + "theoretical_loss": 4.075696797580071, + "tokens_seen": 347996160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045178535606820466, + "loss": 3.0587, + "theoretical_loss": 4.075608916572684, + "tokens_seen": 348061696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517753259779338, + "loss": 2.9124, + "theoretical_loss": 4.075521056742879, + "tokens_seen": 348127232 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451765295887663, + "loss": 3.3636, + "theoretical_loss": 4.0754332180815664, + "tokens_seen": 348192768 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045175526579739215, + "loss": 3.1258, + "theoretical_loss": 4.075345400579664, + "tokens_seen": 348258304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517452357071214, + "loss": 2.8544, + "theoretical_loss": 4.075257604228094, + "tokens_seen": 348323840 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173520561685057, + "loss": 2.979, + "theoretical_loss": 4.075169829017785, + "tokens_seen": 348389376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045172517552657975, + "loss": 2.9919, + "theoretical_loss": 4.07508207493967, + "tokens_seen": 348454912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045171514543630893, + "loss": 3.0188, + "theoretical_loss": 4.074994341984689, + "tokens_seen": 348520448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045170511534603817, + "loss": 2.7695, + "theoretical_loss": 4.074906630143786, + "tokens_seen": 348585984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516950852557673, + "loss": 3.2483, + "theoretical_loss": 4.074818939407912, + "tokens_seen": 348651520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045168505516549653, + "loss": 3.1331, + "theoretical_loss": 4.074731269768024, + "tokens_seen": 348717056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045167502507522566, + "loss": 2.8828, + "theoretical_loss": 4.074643621215082, + "tokens_seen": 348782592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516649949849549, + "loss": 2.8352, + "theoretical_loss": 4.074555993740055, + "tokens_seen": 348848128 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045165496489468407, + "loss": 3.1457, + "theoretical_loss": 4.074468387333914, + "tokens_seen": 348913664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 203975, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.054220199584961, + "objective/train/theoretical_loss": 4.074380801987639, + "objective/train/tokens_used": 369439200, + "theoretical_loss": 4.074380801987639, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045164493480441325, + "loss": 3.0341, + "theoretical_loss": 4.074380801987639, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045163490471414244, + "loss": 3.2664, + "theoretical_loss": 4.074293237692212, + "tokens_seen": 349044736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516248746238716, + "loss": 3.0151, + "theoretical_loss": 4.074205694438623, + "tokens_seen": 349110272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516148445336008, + "loss": 3.0995, + "theoretical_loss": 4.074118172217869, + "tokens_seen": 349175808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045160481444333003, + "loss": 2.9087, + "theoretical_loss": 4.074030671020948, + "tokens_seen": 349241344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045159478435305916, + "loss": 2.9197, + "theoretical_loss": 4.073943190838868, + "tokens_seen": 349306880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515847542627884, + "loss": 2.9025, + "theoretical_loss": 4.073855731662639, + "tokens_seen": 349372416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515747241725175, + "loss": 3.0998, + "theoretical_loss": 4.07376829348328, + "tokens_seen": 349437952 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045156469408224676, + "loss": 3.1268, + "theoretical_loss": 4.073680876291811, + "tokens_seen": 349503488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045155466399197594, + "loss": 2.8534, + "theoretical_loss": 4.073593480079262, + "tokens_seen": 349569024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515446339017051, + "loss": 2.983, + "theoretical_loss": 4.073506104836667, + "tokens_seen": 349634560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515346038114343, + "loss": 3.0149, + "theoretical_loss": 4.073418750555065, + "tokens_seen": 349700096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045152457372116354, + "loss": 3.0564, + "theoretical_loss": 4.073331417225501, + "tokens_seen": 349765632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045151454363089266, + "loss": 2.986, + "theoretical_loss": 4.0732441048390235, + "tokens_seen": 349831168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515045135406219, + "loss": 2.9123, + "theoretical_loss": 4.07315681338669, + "tokens_seen": 349896704 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451494483450351, + "loss": 2.8782, + "theoretical_loss": 4.07306954285956, + "tokens_seen": 349962240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045148445336008026, + "loss": 2.8731, + "theoretical_loss": 4.072982293248703, + "tokens_seen": 350027776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045147442326980944, + "loss": 2.9387, + "theoretical_loss": 4.072895064545189, + "tokens_seen": 350093312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514643931795386, + "loss": 2.9963, + "theoretical_loss": 4.072807856740095, + "tokens_seen": 350158848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514543630892678, + "loss": 3.0698, + "theoretical_loss": 4.072720669824506, + "tokens_seen": 350224384 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451444332998997, + "loss": 2.9614, + "theoretical_loss": 4.072633503789511, + "tokens_seen": 350289920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045143430290872617, + "loss": 3.2203, + "theoretical_loss": 4.072546358626202, + "tokens_seen": 350355456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514242728184554, + "loss": 3.0721, + "theoretical_loss": 4.07245923432568, + "tokens_seen": 350420992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045141424272818453, + "loss": 2.6751, + "theoretical_loss": 4.0723721308790495, + "tokens_seen": 350486528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045140421263791376, + "loss": 2.6701, + "theoretical_loss": 4.072285048277421, + "tokens_seen": 350552064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 205216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9308574199676514, + "objective/train/theoretical_loss": 4.072197986511911, + "objective/train/tokens_used": 371077600, + "theoretical_loss": 4.072197986511911, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045139418254764294, + "loss": 3.0665, + "theoretical_loss": 4.072197986511911, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513841524573721, + "loss": 3.2069, + "theoretical_loss": 4.07211094557364, + "tokens_seen": 350683136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513741223671013, + "loss": 2.7969, + "theoretical_loss": 4.072023925453736, + "tokens_seen": 350748672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513640922768305, + "loss": 3.0486, + "theoretical_loss": 4.07193692614333, + "tokens_seen": 350814208 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045135406218655967, + "loss": 2.8671, + "theoretical_loss": 4.0718499476335595, + "tokens_seen": 350879744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513440320962889, + "loss": 2.9117, + "theoretical_loss": 4.071762989915569, + "tokens_seen": 350945280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045133400200601803, + "loss": 3.066, + "theoretical_loss": 4.071676052980506, + "tokens_seen": 351010816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045132397191574727, + "loss": 3.1371, + "theoretical_loss": 4.071589136819525, + "tokens_seen": 351076352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513139418254764, + "loss": 2.9283, + "theoretical_loss": 4.071502241423786, + "tokens_seen": 351141888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045130391173520563, + "loss": 3.0026, + "theoretical_loss": 4.071415366784452, + "tokens_seen": 351207424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512938816449348, + "loss": 2.7252, + "theoretical_loss": 4.071328512892694, + "tokens_seen": 351272960 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451283851554664, + "loss": 2.9749, + "theoretical_loss": 4.071241679739687, + "tokens_seen": 351338496 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045127382146439317, + "loss": 2.8857, + "theoretical_loss": 4.071154867316614, + "tokens_seen": 351404032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045126379137412235, + "loss": 3.0538, + "theoretical_loss": 4.07106807561466, + "tokens_seen": 351469568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125376128385153, + "loss": 3.0197, + "theoretical_loss": 4.0709813046250165, + "tokens_seen": 351535104 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045124373119358077, + "loss": 2.8479, + "theoretical_loss": 4.070894554338881, + "tokens_seen": 351600640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512337011033099, + "loss": 2.9532, + "theoretical_loss": 4.070807824747456, + "tokens_seen": 351666176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045122367101303913, + "loss": 2.9436, + "theoretical_loss": 4.07072111584195, + "tokens_seen": 351731712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512136409227683, + "loss": 2.8975, + "theoretical_loss": 4.070634427613575, + "tokens_seen": 351797248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512036108324975, + "loss": 2.8211, + "theoretical_loss": 4.070547760053551, + "tokens_seen": 351862784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045119358074222673, + "loss": 2.8698, + "theoretical_loss": 4.070461113153101, + "tokens_seen": 351928320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045118355065195586, + "loss": 2.8615, + "theoretical_loss": 4.0703744869034555, + "tokens_seen": 351993856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511735205616851, + "loss": 2.9633, + "theoretical_loss": 4.070287881295848, + "tokens_seen": 352059392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511634904714143, + "loss": 2.8897, + "theoretical_loss": 4.070201296321519, + "tokens_seen": 352124928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045115346038114345, + "loss": 2.9223, + "theoretical_loss": 4.070114731971715, + "tokens_seen": 352190464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 206163, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.493535280227661, + "objective/train/theoretical_loss": 4.070028188237686, + "objective/train/tokens_used": 372716000, + "theoretical_loss": 4.070028188237686, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045114343029087264, + "loss": 2.8772, + "theoretical_loss": 4.070028188237686, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511334002006018, + "loss": 3.067, + "theoretical_loss": 4.069941665110687, + "tokens_seen": 352321536 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451123370110331, + "loss": 2.9246, + "theoretical_loss": 4.06985516258198, + "tokens_seen": 352387072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045111334002006023, + "loss": 3.006, + "theoretical_loss": 4.069768680642833, + "tokens_seen": 352452608 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045110330992978936, + "loss": 2.9814, + "theoretical_loss": 4.069682219284518, + "tokens_seen": 352518144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510932798395186, + "loss": 3.105, + "theoretical_loss": 4.069595778498311, + "tokens_seen": 352583680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510832497492477, + "loss": 2.8652, + "theoretical_loss": 4.069509358275495, + "tokens_seen": 352649216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045107321965897696, + "loss": 2.854, + "theoretical_loss": 4.069422958607359, + "tokens_seen": 352714752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045106318956870614, + "loss": 2.8667, + "theoretical_loss": 4.069336579485196, + "tokens_seen": 352780288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510531594784353, + "loss": 2.9462, + "theoretical_loss": 4.0692502209003045, + "tokens_seen": 352845824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510431293881645, + "loss": 2.9122, + "theoretical_loss": 4.0691638828439896, + "tokens_seen": 352911360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045103309929789374, + "loss": 2.9916, + "theoretical_loss": 4.069077565307558, + "tokens_seen": 352976896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045102306920762286, + "loss": 2.9217, + "theoretical_loss": 4.068991268282327, + "tokens_seen": 353042432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510130391173521, + "loss": 2.6557, + "theoretical_loss": 4.068904991759615, + "tokens_seen": 353107968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510030090270812, + "loss": 2.8802, + "theoretical_loss": 4.068818735730747, + "tokens_seen": 353173504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045099297893681046, + "loss": 3.0742, + "theoretical_loss": 4.068732500187053, + "tokens_seen": 353239040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045098294884653964, + "loss": 2.9343, + "theoretical_loss": 4.068646285119871, + "tokens_seen": 353304576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509729187562688, + "loss": 2.7554, + "theoretical_loss": 4.06856009052054, + "tokens_seen": 353370112 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450962888665998, + "loss": 3.0227, + "theoretical_loss": 4.068473916380406, + "tokens_seen": 353435648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509528585757272, + "loss": 2.8967, + "theoretical_loss": 4.068387762690822, + "tokens_seen": 353501184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045094282848545637, + "loss": 2.9807, + "theoretical_loss": 4.068301629443143, + "tokens_seen": 353566720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509327983951856, + "loss": 2.9679, + "theoretical_loss": 4.068215516628733, + "tokens_seen": 353632256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045092276830491473, + "loss": 2.9093, + "theoretical_loss": 4.068129424238958, + "tokens_seen": 353697792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045091273821464396, + "loss": 2.9453, + "theoretical_loss": 4.06804335226519, + "tokens_seen": 353763328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045090270812437314, + "loss": 2.6639, + "theoretical_loss": 4.067957300698807, + "tokens_seen": 353828864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 206742, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6503775119781494, + "objective/train/theoretical_loss": 4.067871269531193, + "objective/train/tokens_used": 374354400, + "theoretical_loss": 4.067871269531193, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508926780341023, + "loss": 2.8327, + "theoretical_loss": 4.067871269531193, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508826479438315, + "loss": 2.6181, + "theoretical_loss": 4.0677852587537355, + "tokens_seen": 353959936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508726178535607, + "loss": 3.0139, + "theoretical_loss": 4.067699268357828, + "tokens_seen": 354025472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045086258776328987, + "loss": 2.9987, + "theoretical_loss": 4.0676132983348685, + "tokens_seen": 354091008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508525576730191, + "loss": 3.1759, + "theoretical_loss": 4.067527348676262, + "tokens_seen": 354156544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045084252758274823, + "loss": 2.9902, + "theoretical_loss": 4.067441419373417, + "tokens_seen": 354222080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045083249749247747, + "loss": 2.9154, + "theoretical_loss": 4.067355510417749, + "tokens_seen": 354287616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508224674022066, + "loss": 2.7929, + "theoretical_loss": 4.067269621800675, + "tokens_seen": 354353152 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045081243731193583, + "loss": 2.8677, + "theoretical_loss": 4.067183753513622, + "tokens_seen": 354418688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450802407221665, + "loss": 2.9784, + "theoretical_loss": 4.067097905548019, + "tokens_seen": 354484224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507923771313942, + "loss": 3.0821, + "theoretical_loss": 4.067012077895301, + "tokens_seen": 354549760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045078234704112337, + "loss": 2.7986, + "theoretical_loss": 4.066926270546909, + "tokens_seen": 354615296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045077231695085255, + "loss": 2.842, + "theoretical_loss": 4.066840483494287, + "tokens_seen": 354680832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045076228686058173, + "loss": 2.8722, + "theoretical_loss": 4.066754716728888, + "tokens_seen": 354746368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045075225677031097, + "loss": 2.9914, + "theoretical_loss": 4.066668970242167, + "tokens_seen": 354811904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507422266800401, + "loss": 2.8883, + "theoretical_loss": 4.066583244025583, + "tokens_seen": 354877440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045073219658976933, + "loss": 2.8113, + "theoretical_loss": 4.066497538070605, + "tokens_seen": 354942976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507221664994985, + "loss": 2.591, + "theoretical_loss": 4.066411852368702, + "tokens_seen": 355008512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507121364092277, + "loss": 2.7762, + "theoretical_loss": 4.066326186911353, + "tokens_seen": 355074048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507021063189569, + "loss": 2.7745, + "theoretical_loss": 4.066240541690038, + "tokens_seen": 355139584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045069207622868606, + "loss": 2.7743, + "theoretical_loss": 4.066154916696245, + "tokens_seen": 355205120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045068204613841524, + "loss": 2.9515, + "theoretical_loss": 4.066069311921465, + "tokens_seen": 355270656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506720160481445, + "loss": 2.7834, + "theoretical_loss": 4.065983727357195, + "tokens_seen": 355336192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506619859578736, + "loss": 2.8959, + "theoretical_loss": 4.065898162994938, + "tokens_seen": 355401728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045065195586760284, + "loss": 2.8629, + "theoretical_loss": 4.065812618826202, + "tokens_seen": 355467264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 207983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6451644897460938, + "objective/train/theoretical_loss": 4.065727094842498, + "objective/train/tokens_used": 375992800, + "theoretical_loss": 4.065727094842498, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045064192577733196, + "loss": 2.8682, + "theoretical_loss": 4.065727094842498, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506318956870612, + "loss": 2.7648, + "theoretical_loss": 4.0656415910353445, + "tokens_seen": 355598336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506218655967904, + "loss": 2.9992, + "theoretical_loss": 4.065556107396265, + "tokens_seen": 355663872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045061183550651956, + "loss": 3.2104, + "theoretical_loss": 4.065470643916786, + "tokens_seen": 355729408 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045060180541624874, + "loss": 3.0774, + "theoretical_loss": 4.065385200588442, + "tokens_seen": 355794944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505917753259779, + "loss": 2.9544, + "theoretical_loss": 4.065299777402771, + "tokens_seen": 355860480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505817452357071, + "loss": 2.8752, + "theoretical_loss": 4.065214374351315, + "tokens_seen": 355926016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045057171514543634, + "loss": 2.7962, + "theoretical_loss": 4.065128991425624, + "tokens_seen": 355991552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045056168505516547, + "loss": 2.74, + "theoretical_loss": 4.06504362861725, + "tokens_seen": 356057088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505516549648947, + "loss": 2.8575, + "theoretical_loss": 4.064958285917752, + "tokens_seen": 356122624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505416248746239, + "loss": 2.8417, + "theoretical_loss": 4.064872963318696, + "tokens_seen": 356188160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045053159478435306, + "loss": 3.0414, + "theoretical_loss": 4.064787660811648, + "tokens_seen": 356253696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045052156469408224, + "loss": 2.7343, + "theoretical_loss": 4.064702378388183, + "tokens_seen": 356319232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505115346038114, + "loss": 3.0008, + "theoretical_loss": 4.064617116039878, + "tokens_seen": 356384768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505015045135406, + "loss": 2.9624, + "theoretical_loss": 4.064531873758321, + "tokens_seen": 356450304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045049147442326984, + "loss": 2.9619, + "theoretical_loss": 4.064446651535098, + "tokens_seen": 356515840 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045048144433299897, + "loss": 2.7413, + "theoretical_loss": 4.0643614493618045, + "tokens_seen": 356581376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504714142427282, + "loss": 2.9723, + "theoretical_loss": 4.0642762672300385, + "tokens_seen": 356646912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045046138415245733, + "loss": 2.8596, + "theoretical_loss": 4.064191105131406, + "tokens_seen": 356712448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045045135406218657, + "loss": 2.7957, + "theoretical_loss": 4.064105963057515, + "tokens_seen": 356777984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504413239719158, + "loss": 2.7377, + "theoretical_loss": 4.064020840999981, + "tokens_seen": 356843520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045043129388164493, + "loss": 2.8931, + "theoretical_loss": 4.063935738950423, + "tokens_seen": 356909056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045042126379137416, + "loss": 3.1138, + "theoretical_loss": 4.063850656900465, + "tokens_seen": 356974592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045041123370110334, + "loss": 2.9516, + "theoretical_loss": 4.063765594841737, + "tokens_seen": 357040128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504012036108325, + "loss": 2.9924, + "theoretical_loss": 4.063680552765874, + "tokens_seen": 357105664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 208569, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.472182512283325, + "objective/train/theoretical_loss": 4.063595530664515, + "objective/train/tokens_used": 377631200, + "theoretical_loss": 4.063595530664515, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503911735205617, + "loss": 2.9434, + "theoretical_loss": 4.063595530664515, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503811434302909, + "loss": 2.7141, + "theoretical_loss": 4.063510528529306, + "tokens_seen": 357236736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045037111334002007, + "loss": 2.9001, + "theoretical_loss": 4.063425546351894, + "tokens_seen": 357302272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503610832497493, + "loss": 2.9435, + "theoretical_loss": 4.063340584123936, + "tokens_seen": 357367808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045035105315947843, + "loss": 3.0876, + "theoretical_loss": 4.063255641837092, + "tokens_seen": 357433344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045034102306920767, + "loss": 2.7518, + "theoretical_loss": 4.063170719483025, + "tokens_seen": 357498880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503309929789368, + "loss": 2.7653, + "theoretical_loss": 4.063085817053405, + "tokens_seen": 357564416 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045032096288866603, + "loss": 2.9062, + "theoretical_loss": 4.063000934539909, + "tokens_seen": 357629952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503109327983952, + "loss": 2.7442, + "theoretical_loss": 4.062916071934215, + "tokens_seen": 357695488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503009027081244, + "loss": 3.0829, + "theoretical_loss": 4.062831229228007, + "tokens_seen": 357761024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045029087261785357, + "loss": 2.8354, + "theoretical_loss": 4.062746406412977, + "tokens_seen": 357826560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045028084252758275, + "loss": 2.9435, + "theoretical_loss": 4.062661603480818, + "tokens_seen": 357892096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045027081243731193, + "loss": 2.9386, + "theoretical_loss": 4.06257682042323, + "tokens_seen": 357957632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045026078234704117, + "loss": 2.8088, + "theoretical_loss": 4.0624920572319185, + "tokens_seen": 358023168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502507522567703, + "loss": 2.9035, + "theoretical_loss": 4.062407313898593, + "tokens_seen": 358088704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045024072216649953, + "loss": 2.8364, + "theoretical_loss": 4.062322590414967, + "tokens_seen": 358154240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502306920762287, + "loss": 2.8049, + "theoretical_loss": 4.0622378867727615, + "tokens_seen": 358219776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502206619859579, + "loss": 2.9569, + "theoretical_loss": 4.062153202963701, + "tokens_seen": 358285312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502106318956871, + "loss": 2.8268, + "theoretical_loss": 4.062068538979515, + "tokens_seen": 358350848 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045020060180541626, + "loss": 2.8442, + "theoretical_loss": 4.061983894811938, + "tokens_seen": 358416384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045019057171514544, + "loss": 3.1648, + "theoretical_loss": 4.0618992704527095, + "tokens_seen": 358481920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501805416248747, + "loss": 2.9194, + "theoretical_loss": 4.061814665893573, + "tokens_seen": 358547456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501705115346038, + "loss": 2.9655, + "theoretical_loss": 4.06173008112628, + "tokens_seen": 358612992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045016048144433304, + "loss": 2.8605, + "theoretical_loss": 4.061645516142583, + "tokens_seen": 358678528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045015045135406216, + "loss": 2.7901, + "theoretical_loss": 4.0615609709342415, + "tokens_seen": 358744064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 209910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.162785768508911, + "objective/train/theoretical_loss": 4.061476445493021, + "objective/train/tokens_used": 379269600, + "theoretical_loss": 4.061476445493021, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501404212637914, + "loss": 2.6388, + "theoretical_loss": 4.061476445493021, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501303911735206, + "loss": 2.8801, + "theoretical_loss": 4.06139193981069, + "tokens_seen": 358875136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045012036108324976, + "loss": 3.0025, + "theoretical_loss": 4.061307453879023, + "tokens_seen": 358940672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045011033099297894, + "loss": 2.7736, + "theoretical_loss": 4.061222987689797, + "tokens_seen": 359006208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501003009027081, + "loss": 2.8362, + "theoretical_loss": 4.061138541234799, + "tokens_seen": 359071744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500902708124373, + "loss": 2.6216, + "theoretical_loss": 4.061054114505816, + "tokens_seen": 359137280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045008024072216654, + "loss": 3.0416, + "theoretical_loss": 4.060969707494642, + "tokens_seen": 359202816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045007021063189567, + "loss": 2.8598, + "theoretical_loss": 4.0608853201930755, + "tokens_seen": 359268352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500601805416249, + "loss": 3.0376, + "theoretical_loss": 4.0608009525929205, + "tokens_seen": 359333888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500501504513541, + "loss": 2.9702, + "theoretical_loss": 4.060716604685986, + "tokens_seen": 359399424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045004012036108326, + "loss": 2.911, + "theoretical_loss": 4.0606322764640845, + "tokens_seen": 359464960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045003009027081244, + "loss": 2.6158, + "theoretical_loss": 4.060547967919034, + "tokens_seen": 359530496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500200601805416, + "loss": 2.8483, + "theoretical_loss": 4.06046367904266, + "tokens_seen": 359596032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500100300902708, + "loss": 2.8765, + "theoretical_loss": 4.060379409826788, + "tokens_seen": 359661568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045000000000000004, + "loss": 2.9966, + "theoretical_loss": 4.0602951602632515, + "tokens_seen": 359727104 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044998996990972917, + "loss": 2.9513, + "theoretical_loss": 4.060210930343889, + "tokens_seen": 359792640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499799398194584, + "loss": 3.0588, + "theoretical_loss": 4.060126720060543, + "tokens_seen": 359858176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044996990972918753, + "loss": 2.4906, + "theoretical_loss": 4.060042529405061, + "tokens_seen": 359923712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044995987963891677, + "loss": 2.9925, + "theoretical_loss": 4.059958358369295, + "tokens_seen": 359989248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044994984954864595, + "loss": 2.7972, + "theoretical_loss": 4.0598742069451035, + "tokens_seen": 360054784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044993981945837513, + "loss": 3.0398, + "theoretical_loss": 4.059790075124348, + "tokens_seen": 360120320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499297893681043, + "loss": 2.7946, + "theoretical_loss": 4.059705962898896, + "tokens_seen": 360185856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044991975927783355, + "loss": 2.9623, + "theoretical_loss": 4.059621870260618, + "tokens_seen": 360251392 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044990972918756267, + "loss": 2.9151, + "theoretical_loss": 4.059537797201392, + "tokens_seen": 360316928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498996990972919, + "loss": 2.7803, + "theoretical_loss": 4.059453743713099, + "tokens_seen": 360382464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 210586, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.735008478164673, + "objective/train/theoretical_loss": 4.059369709787625, + "objective/train/tokens_used": 380908000, + "theoretical_loss": 4.059369709787625, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044988966900702103, + "loss": 2.9449, + "theoretical_loss": 4.059369709787625, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044987963891675027, + "loss": 2.8092, + "theoretical_loss": 4.059285695416863, + "tokens_seen": 360513536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044986960882647945, + "loss": 2.7777, + "theoretical_loss": 4.059201700592707, + "tokens_seen": 360579072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044985957873620863, + "loss": 3.0467, + "theoretical_loss": 4.05911772530706, + "tokens_seen": 360644608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498495486459378, + "loss": 3.1316, + "theoretical_loss": 4.059033769551827, + "tokens_seen": 360710144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449839518555667, + "loss": 2.8016, + "theoretical_loss": 4.058949833318917, + "tokens_seen": 360775680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498294884653962, + "loss": 3.0944, + "theoretical_loss": 4.058865916600248, + "tokens_seen": 360841216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498194583751254, + "loss": 2.8891, + "theoretical_loss": 4.058782019387738, + "tokens_seen": 360906752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044980942828485454, + "loss": 2.8118, + "theoretical_loss": 4.058698141673313, + "tokens_seen": 360972288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044979939819458377, + "loss": 2.732, + "theoretical_loss": 4.058614283448903, + "tokens_seen": 361037824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497893681043129, + "loss": 2.844, + "theoretical_loss": 4.0585304447064425, + "tokens_seen": 361103360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044977933801404213, + "loss": 2.8023, + "theoretical_loss": 4.05844662543787, + "tokens_seen": 361168896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497693079237713, + "loss": 2.8618, + "theoretical_loss": 4.058362825635132, + "tokens_seen": 361234432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497592778335005, + "loss": 2.9652, + "theoretical_loss": 4.058279045290176, + "tokens_seen": 361299968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497492477432297, + "loss": 2.7903, + "theoretical_loss": 4.058195284394955, + "tokens_seen": 361365504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497392176529589, + "loss": 2.995, + "theoretical_loss": 4.058111542941429, + "tokens_seen": 361431040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044972918756268804, + "loss": 2.835, + "theoretical_loss": 4.05802782092156, + "tokens_seen": 361496576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497191574724173, + "loss": 2.7238, + "theoretical_loss": 4.057944118327319, + "tokens_seen": 361562112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497091273821464, + "loss": 3.148, + "theoretical_loss": 4.057860435150675, + "tokens_seen": 361627648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044969909729187564, + "loss": 2.6922, + "theoretical_loss": 4.057776771383609, + "tokens_seen": 361693184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496890672016049, + "loss": 2.869, + "theoretical_loss": 4.057693127018101, + "tokens_seen": 361758720 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449679037111334, + "loss": 3.0336, + "theoretical_loss": 4.057609502046141, + "tokens_seen": 361824256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044966900702106324, + "loss": 3.0169, + "theoretical_loss": 4.057525896459719, + "tokens_seen": 361889792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044965897693079236, + "loss": 2.8861, + "theoretical_loss": 4.057442310250831, + "tokens_seen": 361955328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496489468405216, + "loss": 2.9571, + "theoretical_loss": 4.0573587434114815, + "tokens_seen": 362020864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 211693, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5061984062194824, + "objective/train/theoretical_loss": 4.057275195933674, + "objective/train/tokens_used": 382546400, + "theoretical_loss": 4.057275195933674, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496389167502508, + "loss": 3.2087, + "theoretical_loss": 4.057275195933674, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044962888665997996, + "loss": 2.835, + "theoretical_loss": 4.057191667809421, + "tokens_seen": 362151936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044961885656970914, + "loss": 2.8273, + "theoretical_loss": 4.057108159030737, + "tokens_seen": 362217472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496088264794383, + "loss": 2.9131, + "theoretical_loss": 4.057024669589644, + "tokens_seen": 362283008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495987963891675, + "loss": 2.7972, + "theoretical_loss": 4.056941199478166, + "tokens_seen": 362348544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044958876629889674, + "loss": 3.1665, + "theoretical_loss": 4.0568577486883335, + "tokens_seen": 362414080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044957873620862587, + "loss": 2.9409, + "theoretical_loss": 4.056774317212181, + "tokens_seen": 362479616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495687061183551, + "loss": 2.6666, + "theoretical_loss": 4.056690905041748, + "tokens_seen": 362545152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495586760280843, + "loss": 2.8504, + "theoretical_loss": 4.056607512169078, + "tokens_seen": 362610688 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044954864593781346, + "loss": 2.8855, + "theoretical_loss": 4.056524138586221, + "tokens_seen": 362676224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044953861584754264, + "loss": 2.654, + "theoretical_loss": 4.056440784285231, + "tokens_seen": 362741760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495285857572718, + "loss": 2.9835, + "theoretical_loss": 4.056357449258163, + "tokens_seen": 362807296 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449518555667001, + "loss": 2.9323, + "theoretical_loss": 4.056274133497084, + "tokens_seen": 362872832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044950852557673024, + "loss": 2.8978, + "theoretical_loss": 4.056190836994059, + "tokens_seen": 362938368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044949849548645937, + "loss": 2.6167, + "theoretical_loss": 4.056107559741161, + "tokens_seen": 363003904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494884653961886, + "loss": 2.942, + "theoretical_loss": 4.056024301730467, + "tokens_seen": 363069440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044947843530591773, + "loss": 2.9565, + "theoretical_loss": 4.05594106295406, + "tokens_seen": 363134976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044946840521564697, + "loss": 2.7726, + "theoretical_loss": 4.055857843404025, + "tokens_seen": 363200512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044945837512537615, + "loss": 3.0216, + "theoretical_loss": 4.0557746430724535, + "tokens_seen": 363266048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044944834503510533, + "loss": 3.1954, + "theoretical_loss": 4.05569146195144, + "tokens_seen": 363331584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494383149448345, + "loss": 2.9553, + "theoretical_loss": 4.055608300033088, + "tokens_seen": 363397120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044942828485456375, + "loss": 3.024, + "theoretical_loss": 4.0555251573095, + "tokens_seen": 363462656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044941825476429287, + "loss": 2.7395, + "theoretical_loss": 4.0554420337727874, + "tokens_seen": 363528192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494082246740221, + "loss": 2.8736, + "theoretical_loss": 4.055358929415063, + "tokens_seen": 363593728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044939819458375123, + "loss": 2.8492, + "theoretical_loss": 4.055275844228448, + "tokens_seen": 363659264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 212305, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9765894412994385, + "objective/train/theoretical_loss": 4.055192778205064, + "objective/train/tokens_used": 384184800, + "theoretical_loss": 4.055192778205064, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044938816449348047, + "loss": 3.0477, + "theoretical_loss": 4.055192778205064, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044937813440320965, + "loss": 2.7255, + "theoretical_loss": 4.0551097313370414, + "tokens_seen": 363790336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044936810431293883, + "loss": 2.974, + "theoretical_loss": 4.055026703616512, + "tokens_seen": 363855872 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449358074222668, + "loss": 2.9851, + "theoretical_loss": 4.054943695035614, + "tokens_seen": 363921408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493480441323972, + "loss": 3.114, + "theoretical_loss": 4.054860705586489, + "tokens_seen": 363986944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493380140421264, + "loss": 2.9801, + "theoretical_loss": 4.054777735261286, + "tokens_seen": 364052480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493279839518556, + "loss": 2.8191, + "theoretical_loss": 4.054694784052155, + "tokens_seen": 364118016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044931795386158474, + "loss": 2.8661, + "theoretical_loss": 4.054611851951253, + "tokens_seen": 364183552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044930792377131397, + "loss": 2.7798, + "theoretical_loss": 4.054528938950741, + "tokens_seen": 364249088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492978936810431, + "loss": 2.9318, + "theoretical_loss": 4.054446045042783, + "tokens_seen": 364314624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044928786359077234, + "loss": 3.0197, + "theoretical_loss": 4.054363170219551, + "tokens_seen": 364380160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492778335005015, + "loss": 2.8279, + "theoretical_loss": 4.054280314473219, + "tokens_seen": 364445696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492678034102307, + "loss": 2.8172, + "theoretical_loss": 4.054197477795967, + "tokens_seen": 364511232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492577733199599, + "loss": 3.0234, + "theoretical_loss": 4.0541146601799785, + "tokens_seen": 364576768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492477432296891, + "loss": 2.675, + "theoretical_loss": 4.0540318616174424, + "tokens_seen": 364642304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044923771313941824, + "loss": 2.6453, + "theoretical_loss": 4.0539490821005515, + "tokens_seen": 364707840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492276830491475, + "loss": 2.7324, + "theoretical_loss": 4.053866321621504, + "tokens_seen": 364773376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492176529588766, + "loss": 2.8083, + "theoretical_loss": 4.053783580172501, + "tokens_seen": 364838912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044920762286860584, + "loss": 2.8838, + "theoretical_loss": 4.053700857745753, + "tokens_seen": 364904448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449197592778335, + "loss": 2.492, + "theoretical_loss": 4.053618154333467, + "tokens_seen": 364969984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491875626880642, + "loss": 2.869, + "theoretical_loss": 4.053535469927864, + "tokens_seen": 365035520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491775325977934, + "loss": 2.8246, + "theoretical_loss": 4.05345280452116, + "tokens_seen": 365101056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044916750250752256, + "loss": 2.7243, + "theoretical_loss": 4.053370158105584, + "tokens_seen": 365166592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044915747241725174, + "loss": 2.5572, + "theoretical_loss": 4.053287530673364, + "tokens_seen": 365232128 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449147442326981, + "loss": 2.9422, + "theoretical_loss": 4.053204922216736, + "tokens_seen": 365297664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 213520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0496366024017334, + "objective/train/theoretical_loss": 4.053122332727937, + "objective/train/tokens_used": 385823200, + "theoretical_loss": 4.053122332727937, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491374122367101, + "loss": 2.9336, + "theoretical_loss": 4.053122332727937, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044912738214643934, + "loss": 2.6247, + "theoretical_loss": 4.053039762199212, + "tokens_seen": 365428736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044911735205616847, + "loss": 2.6713, + "theoretical_loss": 4.05295721062281, + "tokens_seen": 365494272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491073219658977, + "loss": 3.0444, + "theoretical_loss": 4.0528746779909826, + "tokens_seen": 365559808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490972918756269, + "loss": 2.8455, + "theoretical_loss": 4.052792164295987, + "tokens_seen": 365625344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044908726178535607, + "loss": 2.7943, + "theoretical_loss": 4.052709669530085, + "tokens_seen": 365690880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044907723169508525, + "loss": 2.8501, + "theoretical_loss": 4.052627193685544, + "tokens_seen": 365756416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490672016048145, + "loss": 2.8558, + "theoretical_loss": 4.052544736754634, + "tokens_seen": 365821952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490571715145436, + "loss": 2.9659, + "theoretical_loss": 4.0524622987296315, + "tokens_seen": 365887488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044904714142427284, + "loss": 2.8449, + "theoretical_loss": 4.052379879602816, + "tokens_seen": 365953024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044903711133400197, + "loss": 2.8962, + "theoretical_loss": 4.052297479366471, + "tokens_seen": 366018560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490270812437312, + "loss": 2.6586, + "theoretical_loss": 4.052215098012887, + "tokens_seen": 366084096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490170511534604, + "loss": 2.6115, + "theoretical_loss": 4.052132735534358, + "tokens_seen": 366149632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044900702106318957, + "loss": 2.5501, + "theoretical_loss": 4.0520503919231805, + "tokens_seen": 366215168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044899699097291875, + "loss": 2.7229, + "theoretical_loss": 4.051968067171659, + "tokens_seen": 366280704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044898696088264793, + "loss": 2.66, + "theoretical_loss": 4.051885761272098, + "tokens_seen": 366346240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489769307923771, + "loss": 2.8702, + "theoretical_loss": 4.051803474216813, + "tokens_seen": 366411776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044896690070210635, + "loss": 3.0329, + "theoretical_loss": 4.051721205998117, + "tokens_seen": 366477312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489568706118355, + "loss": 3.0362, + "theoretical_loss": 4.051638956608333, + "tokens_seen": 366542848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489468405215647, + "loss": 3.0241, + "theoretical_loss": 4.0515567260397845, + "tokens_seen": 366608384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044893681043129395, + "loss": 3.1542, + "theoretical_loss": 4.051474514284801, + "tokens_seen": 366673920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044892678034102307, + "loss": 2.9845, + "theoretical_loss": 4.051392321335719, + "tokens_seen": 366739456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489167502507523, + "loss": 2.8097, + "theoretical_loss": 4.051310147184875, + "tokens_seen": 366804992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044890672016048143, + "loss": 2.7092, + "theoretical_loss": 4.0512279918246135, + "tokens_seen": 366870528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044889669007021067, + "loss": 2.7877, + "theoretical_loss": 4.051145855247281, + "tokens_seen": 366936064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 214095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.887453317642212, + "objective/train/theoretical_loss": 4.051063737445231, + "objective/train/tokens_used": 387461600, + "theoretical_loss": 4.051063737445231, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044888665997993985, + "loss": 3.0162, + "theoretical_loss": 4.051063737445231, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044887662988966903, + "loss": 2.7889, + "theoretical_loss": 4.0509816384108195, + "tokens_seen": 367067136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488665997993982, + "loss": 2.966, + "theoretical_loss": 4.050899558136408, + "tokens_seen": 367132672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488565697091274, + "loss": 2.8294, + "theoretical_loss": 4.050817496614361, + "tokens_seen": 367198208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488465396188566, + "loss": 2.916, + "theoretical_loss": 4.050735453837049, + "tokens_seen": 367263744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488365095285858, + "loss": 2.8996, + "theoretical_loss": 4.0506534297968475, + "tokens_seen": 367329280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044882647943831494, + "loss": 2.8554, + "theoretical_loss": 4.050571424486135, + "tokens_seen": 367394816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488164493480442, + "loss": 2.7958, + "theoretical_loss": 4.050489437897295, + "tokens_seen": 367460352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488064192577733, + "loss": 2.9325, + "theoretical_loss": 4.050407470022715, + "tokens_seen": 367525888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044879638916750254, + "loss": 2.9501, + "theoretical_loss": 4.050325520854788, + "tokens_seen": 367591424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487863590772317, + "loss": 2.6799, + "theoretical_loss": 4.050243590385909, + "tokens_seen": 367656960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487763289869609, + "loss": 2.9033, + "theoretical_loss": 4.0501616786084815, + "tokens_seen": 367722496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487662988966901, + "loss": 2.8158, + "theoretical_loss": 4.05007978551491, + "tokens_seen": 367788032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487562688064193, + "loss": 2.8058, + "theoretical_loss": 4.049997911097606, + "tokens_seen": 367853568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044874623871614844, + "loss": 2.7876, + "theoretical_loss": 4.049916055348982, + "tokens_seen": 367919104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487362086258777, + "loss": 3.0632, + "theoretical_loss": 4.049834218261458, + "tokens_seen": 367984640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487261785356068, + "loss": 3.0815, + "theoretical_loss": 4.049752399827458, + "tokens_seen": 368050176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044871614844533604, + "loss": 2.7292, + "theoretical_loss": 4.049670600039409, + "tokens_seen": 368115712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487061183550652, + "loss": 2.8603, + "theoretical_loss": 4.049588818889744, + "tokens_seen": 368181248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486960882647944, + "loss": 3.1734, + "theoretical_loss": 4.049507056370899, + "tokens_seen": 368246784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486860581745236, + "loss": 3.0132, + "theoretical_loss": 4.049425312475315, + "tokens_seen": 368312320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044867602808425276, + "loss": 2.9497, + "theoretical_loss": 4.049343587195439, + "tokens_seen": 368377856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044866599799398194, + "loss": 2.9889, + "theoretical_loss": 4.049261880523719, + "tokens_seen": 368443392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486559679037112, + "loss": 2.7858, + "theoretical_loss": 4.04918019245261, + "tokens_seen": 368508928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486459378134403, + "loss": 2.7187, + "theoretical_loss": 4.049098522974571, + "tokens_seen": 368574464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 215091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5288538932800293, + "objective/train/theoretical_loss": 4.049016872082065, + "objective/train/tokens_used": 389100000, + "theoretical_loss": 4.049016872082065, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044863590772316954, + "loss": 2.8003, + "theoretical_loss": 4.049016872082065, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044862587763289867, + "loss": 3.0327, + "theoretical_loss": 4.04893523976756, + "tokens_seen": 368705536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486158475426279, + "loss": 2.9907, + "theoretical_loss": 4.0488536260235275, + "tokens_seen": 368771072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486058174523571, + "loss": 2.9698, + "theoretical_loss": 4.048772030842443, + "tokens_seen": 368836608 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044859578736208627, + "loss": 3.0078, + "theoretical_loss": 4.048690454216788, + "tokens_seen": 368902144 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044858575727181545, + "loss": 2.8499, + "theoretical_loss": 4.048608896139048, + "tokens_seen": 368967680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485757271815447, + "loss": 3.1026, + "theoretical_loss": 4.048527356601712, + "tokens_seen": 369033216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485656970912738, + "loss": 2.9261, + "theoretical_loss": 4.048445835597272, + "tokens_seen": 369098752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044855566700100304, + "loss": 2.9621, + "theoretical_loss": 4.04836433311823, + "tokens_seen": 369164288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044854563691073217, + "loss": 2.8368, + "theoretical_loss": 4.0482828491570855, + "tokens_seen": 369229824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485356068204614, + "loss": 2.7932, + "theoretical_loss": 4.048201383706347, + "tokens_seen": 369295360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485255767301906, + "loss": 2.7461, + "theoretical_loss": 4.048119936758525, + "tokens_seen": 369360896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044851554663991977, + "loss": 2.9926, + "theoretical_loss": 4.048038508306135, + "tokens_seen": 369426432 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044850551654964895, + "loss": 2.8286, + "theoretical_loss": 4.047957098341698, + "tokens_seen": 369491968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044849548645937813, + "loss": 2.9703, + "theoretical_loss": 4.047875706857737, + "tokens_seen": 369557504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484854563691073, + "loss": 2.7894, + "theoretical_loss": 4.047794333846781, + "tokens_seen": 369623040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044847542627883655, + "loss": 2.9906, + "theoretical_loss": 4.047712979301364, + "tokens_seen": 369688576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484653961885657, + "loss": 2.8056, + "theoretical_loss": 4.047631643214023, + "tokens_seen": 369754112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484553660982949, + "loss": 2.4839, + "theoretical_loss": 4.0475503255772995, + "tokens_seen": 369819648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044844533600802404, + "loss": 2.7831, + "theoretical_loss": 4.047469026383739, + "tokens_seen": 369885184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044843530591775327, + "loss": 2.9546, + "theoretical_loss": 4.047387745625892, + "tokens_seen": 369950720 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044842527582748245, + "loss": 2.7135, + "theoretical_loss": 4.047306483296315, + "tokens_seen": 370016256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044841524573721163, + "loss": 2.861, + "theoretical_loss": 4.047225239387565, + "tokens_seen": 370081792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484052156469408, + "loss": 2.876, + "theoretical_loss": 4.047144013892206, + "tokens_seen": 370147328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044839518555667005, + "loss": 2.8395, + "theoretical_loss": 4.047062806802805, + "tokens_seen": 370212864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 215564, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0313522815704346, + "objective/train/theoretical_loss": 4.046981618111936, + "objective/train/tokens_used": 390738400, + "theoretical_loss": 4.046981618111936, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483851554663992, + "loss": 2.9527, + "theoretical_loss": 4.046981618111936, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483751253761284, + "loss": 2.9376, + "theoretical_loss": 4.046900447812173, + "tokens_seen": 370343936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044836509528585754, + "loss": 2.9443, + "theoretical_loss": 4.046819295896098, + "tokens_seen": 370409472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483550651955868, + "loss": 3.0019, + "theoretical_loss": 4.046738162356295, + "tokens_seen": 370475008 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044834503510531596, + "loss": 2.8873, + "theoretical_loss": 4.046657047185354, + "tokens_seen": 370540544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044833500501504514, + "loss": 3.11, + "theoretical_loss": 4.0465759503758685, + "tokens_seen": 370606080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483249749247743, + "loss": 2.805, + "theoretical_loss": 4.046494871920436, + "tokens_seen": 370671616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483149448345035, + "loss": 2.853, + "theoretical_loss": 4.046413811811659, + "tokens_seen": 370737152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483049147442327, + "loss": 3.0758, + "theoretical_loss": 4.0463327700421425, + "tokens_seen": 370802688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482948846539619, + "loss": 2.7516, + "theoretical_loss": 4.046251746604499, + "tokens_seen": 370868224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828485456369104, + "loss": 2.9978, + "theoretical_loss": 4.046170741491343, + "tokens_seen": 370933760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482748244734203, + "loss": 2.7836, + "theoretical_loss": 4.046089754695292, + "tokens_seen": 370999296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482647943831494, + "loss": 2.7176, + "theoretical_loss": 4.046008786208972, + "tokens_seen": 371064832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044825476429287864, + "loss": 2.9292, + "theoretical_loss": 4.04592783602501, + "tokens_seen": 371130368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482447342026078, + "loss": 2.9083, + "theoretical_loss": 4.045846904136037, + "tokens_seen": 371195904 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448234704112337, + "loss": 2.6722, + "theoretical_loss": 4.04576599053469, + "tokens_seen": 371261440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482246740220662, + "loss": 3.207, + "theoretical_loss": 4.045685095213611, + "tokens_seen": 371326976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482146439317954, + "loss": 2.7582, + "theoretical_loss": 4.045604218165442, + "tokens_seen": 371392512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044820461384152455, + "loss": 2.6807, + "theoretical_loss": 4.045523359382835, + "tokens_seen": 371458048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481945837512538, + "loss": 3.2028, + "theoretical_loss": 4.045442518858441, + "tokens_seen": 371523584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044818455366098296, + "loss": 2.8334, + "theoretical_loss": 4.045361696584919, + "tokens_seen": 371589120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044817452357071214, + "loss": 2.8413, + "theoretical_loss": 4.04528089255493, + "tokens_seen": 371654656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481644934804414, + "loss": 3.0689, + "theoretical_loss": 4.045200106761142, + "tokens_seen": 371720192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481544633901705, + "loss": 3.1544, + "theoretical_loss": 4.045119339196222, + "tokens_seen": 371785728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044814443329989974, + "loss": 2.6545, + "theoretical_loss": 4.045038589852847, + "tokens_seen": 371851264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 216610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.039722442626953, + "objective/train/theoretical_loss": 4.044957858723696, + "objective/train/tokens_used": 392376800, + "theoretical_loss": 4.044957858723696, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044813440320962887, + "loss": 2.9195, + "theoretical_loss": 4.044957858723696, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481243731193581, + "loss": 2.9107, + "theoretical_loss": 4.0448771458014505, + "tokens_seen": 371982336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481143430290873, + "loss": 2.8144, + "theoretical_loss": 4.044796451078799, + "tokens_seen": 372047872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044810431293881647, + "loss": 2.8264, + "theoretical_loss": 4.044715774548433, + "tokens_seen": 372113408 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044809428284854565, + "loss": 2.7547, + "theoretical_loss": 4.044635116203047, + "tokens_seen": 372178944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480842527582749, + "loss": 2.8847, + "theoretical_loss": 4.0445544760353425, + "tokens_seen": 372244480 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448074222668004, + "loss": 2.7869, + "theoretical_loss": 4.044473854038022, + "tokens_seen": 372310016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044806419257773324, + "loss": 2.9045, + "theoretical_loss": 4.044393250203795, + "tokens_seen": 372375552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044805416248746237, + "loss": 3.1393, + "theoretical_loss": 4.044312664525374, + "tokens_seen": 372441088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480441323971916, + "loss": 2.9725, + "theoretical_loss": 4.044232096995476, + "tokens_seen": 372506624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480341023069208, + "loss": 2.984, + "theoretical_loss": 4.044151547606821, + "tokens_seen": 372572160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044802407221664997, + "loss": 2.9168, + "theoretical_loss": 4.0440710163521345, + "tokens_seen": 372637696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044801404212637915, + "loss": 2.9977, + "theoretical_loss": 4.043990503224146, + "tokens_seen": 372703232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044800401203610833, + "loss": 3.0303, + "theoretical_loss": 4.04391000821559, + "tokens_seen": 372768768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479939819458375, + "loss": 3.2721, + "theoretical_loss": 4.0438295313192025, + "tokens_seen": 372834304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044798395185556675, + "loss": 2.6246, + "theoretical_loss": 4.043749072527727, + "tokens_seen": 372899840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479739217652959, + "loss": 2.7876, + "theoretical_loss": 4.043668631833908, + "tokens_seen": 372965376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479638916750251, + "loss": 2.6676, + "theoretical_loss": 4.043588209230498, + "tokens_seen": 373030912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044795386158475424, + "loss": 2.7663, + "theoretical_loss": 4.04350780471025, + "tokens_seen": 373096448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044794383149448347, + "loss": 2.9916, + "theoretical_loss": 4.043427418265923, + "tokens_seen": 373161984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044793380140421265, + "loss": 2.8541, + "theoretical_loss": 4.043347049890279, + "tokens_seen": 373227520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044792377131394183, + "loss": 2.8811, + "theoretical_loss": 4.043266699576086, + "tokens_seen": 373293056 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447913741223671, + "loss": 2.8581, + "theoretical_loss": 4.043186367316116, + "tokens_seen": 373358592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044790371113340025, + "loss": 2.804, + "theoretical_loss": 4.043106053103141, + "tokens_seen": 373424128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478936810431294, + "loss": 2.9816, + "theoretical_loss": 4.043025756929945, + "tokens_seen": 373489664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 217105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.010817766189575, + "objective/train/theoretical_loss": 4.042945478789308, + "objective/train/tokens_used": 394015200, + "theoretical_loss": 4.042945478789308, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478836509528586, + "loss": 3.1259, + "theoretical_loss": 4.042945478789308, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044787362086258774, + "loss": 3.0529, + "theoretical_loss": 4.042865218674019, + "tokens_seen": 373620736 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447863590772317, + "loss": 2.7388, + "theoretical_loss": 4.04278497657687, + "tokens_seen": 373686272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044785356068204616, + "loss": 2.8478, + "theoretical_loss": 4.042704752490658, + "tokens_seen": 373751808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044784353059177534, + "loss": 2.7947, + "theoretical_loss": 4.042624546408182, + "tokens_seen": 373817344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478335005015045, + "loss": 2.8891, + "theoretical_loss": 4.042544358322247, + "tokens_seen": 373882880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478234704112337, + "loss": 2.9725, + "theoretical_loss": 4.042464188225661, + "tokens_seen": 373948416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478134403209629, + "loss": 2.9604, + "theoretical_loss": 4.042384036111236, + "tokens_seen": 374013952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478034102306921, + "loss": 2.8276, + "theoretical_loss": 4.04230390197179, + "tokens_seen": 374079488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044779338014042124, + "loss": 3.1022, + "theoretical_loss": 4.042223785800144, + "tokens_seen": 374145024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477833500501505, + "loss": 3.1569, + "theoretical_loss": 4.042143687589123, + "tokens_seen": 374210560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477733199598796, + "loss": 2.868, + "theoretical_loss": 4.042063607331555, + "tokens_seen": 374276096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044776328986960884, + "loss": 2.8411, + "theoretical_loss": 4.0419835450202735, + "tokens_seen": 374341632 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447753259779338, + "loss": 3.0814, + "theoretical_loss": 4.041903500648116, + "tokens_seen": 374407168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477432296890672, + "loss": 2.9514, + "theoretical_loss": 4.041823474207925, + "tokens_seen": 374472704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477331995987964, + "loss": 3.0362, + "theoretical_loss": 4.041743465692545, + "tokens_seen": 374538240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477231695085256, + "loss": 2.8953, + "theoretical_loss": 4.041663475094826, + "tokens_seen": 374603776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044771313941825475, + "loss": 2.8501, + "theoretical_loss": 4.041583502407622, + "tokens_seen": 374669312 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447703109327984, + "loss": 3.0705, + "theoretical_loss": 4.041503547623791, + "tokens_seen": 374734848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476930792377131, + "loss": 2.8879, + "theoretical_loss": 4.041423610736195, + "tokens_seen": 374800384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044768304914744234, + "loss": 2.801, + "theoretical_loss": 4.0413436917376995, + "tokens_seen": 374865920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476730190571715, + "loss": 2.9406, + "theoretical_loss": 4.041263790621175, + "tokens_seen": 374931456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476629889669007, + "loss": 2.859, + "theoretical_loss": 4.041183907379495, + "tokens_seen": 374996992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476529588766299, + "loss": 2.9117, + "theoretical_loss": 4.041104042005539, + "tokens_seen": 375062528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044764292878635907, + "loss": 3.1132, + "theoretical_loss": 4.04102419449219, + "tokens_seen": 375128064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 218363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9804539680480957, + "objective/train/theoretical_loss": 4.040944364832332, + "objective/train/tokens_used": 395653600, + "theoretical_loss": 4.040944364832332, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044763289869608825, + "loss": 2.9225, + "theoretical_loss": 4.040944364832332, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476228686058175, + "loss": 2.854, + "theoretical_loss": 4.040864553018858, + "tokens_seen": 375259136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476128385155466, + "loss": 2.7824, + "theoretical_loss": 4.040784759044662, + "tokens_seen": 375324672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044760280842527585, + "loss": 2.8757, + "theoretical_loss": 4.040704982902642, + "tokens_seen": 375390208 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044759277833500503, + "loss": 2.9178, + "theoretical_loss": 4.0406252245857015, + "tokens_seen": 375455744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475827482447342, + "loss": 3.1065, + "theoretical_loss": 4.0405454840867465, + "tokens_seen": 375521280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475727181544634, + "loss": 2.7781, + "theoretical_loss": 4.040465761398689, + "tokens_seen": 375586816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044756268806419257, + "loss": 2.9638, + "theoretical_loss": 4.0403860565144445, + "tokens_seen": 375652352 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044755265797392175, + "loss": 3.18, + "theoretical_loss": 4.040306369426929, + "tokens_seen": 375717888 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447542627883651, + "loss": 2.7766, + "theoretical_loss": 4.04022670012907, + "tokens_seen": 375783424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475325977933801, + "loss": 2.9218, + "theoretical_loss": 4.0401470486137905, + "tokens_seen": 375848960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044752256770310935, + "loss": 3.0006, + "theoretical_loss": 4.040067414874024, + "tokens_seen": 375914496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475125376128385, + "loss": 2.8125, + "theoretical_loss": 4.039987798902705, + "tokens_seen": 375980032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475025075225677, + "loss": 3.1362, + "theoretical_loss": 4.039908200692773, + "tokens_seen": 376045568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474924774322969, + "loss": 2.9677, + "theoretical_loss": 4.039828620237171, + "tokens_seen": 376111104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474824473420261, + "loss": 2.9863, + "theoretical_loss": 4.039749057528846, + "tokens_seen": 376176640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044747241725175526, + "loss": 2.858, + "theoretical_loss": 4.039669512560751, + "tokens_seen": 376242176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044746238716148444, + "loss": 2.9564, + "theoretical_loss": 4.039589985325839, + "tokens_seen": 376307712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474523570712136, + "loss": 2.976, + "theoretical_loss": 4.039510475817071, + "tokens_seen": 376373248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044744232698094285, + "loss": 3.026, + "theoretical_loss": 4.03943098402741, + "tokens_seen": 376438784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044743229689067204, + "loss": 2.7752, + "theoretical_loss": 4.039351509949824, + "tokens_seen": 376504320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474222668004012, + "loss": 2.9442, + "theoretical_loss": 4.0392720535772835, + "tokens_seen": 376569856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044741223671013045, + "loss": 3.1013, + "theoretical_loss": 4.039192614902763, + "tokens_seen": 376635392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474022066198596, + "loss": 2.8631, + "theoretical_loss": 4.0391131939192455, + "tokens_seen": 376700928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473921765295888, + "loss": 2.9903, + "theoretical_loss": 4.039033790619711, + "tokens_seen": 376766464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 218865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9096364974975586, + "objective/train/theoretical_loss": 4.038954404997148, + "objective/train/tokens_used": 397292000, + "theoretical_loss": 4.038954404997148, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044738214643931794, + "loss": 3.0827, + "theoretical_loss": 4.038954404997148, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473721163490472, + "loss": 3.2042, + "theoretical_loss": 4.038875037044549, + "tokens_seen": 376897536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044736208625877636, + "loss": 3.0121, + "theoretical_loss": 4.038795686754908, + "tokens_seen": 376963072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044735205616850554, + "loss": 2.8018, + "theoretical_loss": 4.038716354121226, + "tokens_seen": 377028608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473420260782347, + "loss": 2.963, + "theoretical_loss": 4.038637039136505, + "tokens_seen": 377094144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473319959879639, + "loss": 2.6641, + "theoretical_loss": 4.0385577417937535, + "tokens_seen": 377159680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473219658976931, + "loss": 3.1323, + "theoretical_loss": 4.0384784620859815, + "tokens_seen": 377225216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473119358074223, + "loss": 2.9769, + "theoretical_loss": 4.038399200006207, + "tokens_seen": 377290752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044730190571715144, + "loss": 2.9038, + "theoretical_loss": 4.038319955547447, + "tokens_seen": 377356288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472918756268807, + "loss": 2.8752, + "theoretical_loss": 4.038240728702725, + "tokens_seen": 377421824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472818455366098, + "loss": 3.0864, + "theoretical_loss": 4.03816151946507, + "tokens_seen": 377487360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044727181544633904, + "loss": 2.8703, + "theoretical_loss": 4.038082327827512, + "tokens_seen": 377552896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472617853560682, + "loss": 2.896, + "theoretical_loss": 4.038003153783086, + "tokens_seen": 377618432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472517552657974, + "loss": 3.0818, + "theoretical_loss": 4.037923997324833, + "tokens_seen": 377683968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472417251755266, + "loss": 2.9379, + "theoretical_loss": 4.037844858445794, + "tokens_seen": 377749504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472316950852558, + "loss": 2.9901, + "theoretical_loss": 4.037765737139018, + "tokens_seen": 377815040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044722166499498495, + "loss": 2.9231, + "theoretical_loss": 4.037686633397555, + "tokens_seen": 377880576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472116349047142, + "loss": 2.7279, + "theoretical_loss": 4.03760754721446, + "tokens_seen": 377946112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472016048144433, + "loss": 2.8789, + "theoretical_loss": 4.037528478582793, + "tokens_seen": 378011648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044719157472417254, + "loss": 2.8981, + "theoretical_loss": 4.037449427495615, + "tokens_seen": 378077184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471815446339017, + "loss": 3.0708, + "theoretical_loss": 4.037370393945996, + "tokens_seen": 378142720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471715145436309, + "loss": 2.9477, + "theoretical_loss": 4.037291377927005, + "tokens_seen": 378208256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471614844533601, + "loss": 2.8476, + "theoretical_loss": 4.0372123794317165, + "tokens_seen": 378273792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044715145436308927, + "loss": 3.0008, + "theoretical_loss": 4.037133398453209, + "tokens_seen": 378339328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044714142427281845, + "loss": 2.7895, + "theoretical_loss": 4.037054434984566, + "tokens_seen": 378404864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 220023, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9392900466918945, + "objective/train/theoretical_loss": 4.036975489018874, + "objective/train/tokens_used": 398930400, + "theoretical_loss": 4.036975489018874, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471313941825477, + "loss": 2.8371, + "theoretical_loss": 4.036975489018874, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471213640922768, + "loss": 3.062, + "theoretical_loss": 4.036896560549224, + "tokens_seen": 378535936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044711133400200605, + "loss": 2.847, + "theoretical_loss": 4.03681764956871, + "tokens_seen": 378601472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044710130391173523, + "loss": 2.8412, + "theoretical_loss": 4.036738756070429, + "tokens_seen": 378667008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470912738214644, + "loss": 2.7814, + "theoretical_loss": 4.036659880047486, + "tokens_seen": 378732544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470812437311936, + "loss": 2.9737, + "theoretical_loss": 4.036581021492984, + "tokens_seen": 378798080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044707121364092277, + "loss": 2.8625, + "theoretical_loss": 4.036502180400036, + "tokens_seen": 378863616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044706118355065195, + "loss": 2.8385, + "theoretical_loss": 4.036423356761754, + "tokens_seen": 378929152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470511534603812, + "loss": 2.8222, + "theoretical_loss": 4.036344550571257, + "tokens_seen": 378994688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470411233701103, + "loss": 2.8922, + "theoretical_loss": 4.0362657618216655, + "tokens_seen": 379060224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044703109327983955, + "loss": 2.5815, + "theoretical_loss": 4.036186990506107, + "tokens_seen": 379125760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470210631895687, + "loss": 2.9568, + "theoretical_loss": 4.036108236617711, + "tokens_seen": 379191296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470110330992979, + "loss": 2.9341, + "theoretical_loss": 4.036029500149609, + "tokens_seen": 379256832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470010030090271, + "loss": 2.8396, + "theoretical_loss": 4.03595078109494, + "tokens_seen": 379322368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004469909729187563, + "loss": 3.1731, + "theoretical_loss": 4.035872079446846, + "tokens_seen": 379387904 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044698094282848546, + "loss": 2.8303, + "theoretical_loss": 4.03579339519847, + "tokens_seen": 379453440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044697091273821464, + "loss": 2.9496, + "theoretical_loss": 4.035714728342962, + "tokens_seen": 379518976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469608826479438, + "loss": 2.9665, + "theoretical_loss": 4.035636078873477, + "tokens_seen": 379584512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044695085255767305, + "loss": 2.8545, + "theoretical_loss": 4.035557446783168, + "tokens_seen": 379650048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469408224674022, + "loss": 3.0728, + "theoretical_loss": 4.035478832065198, + "tokens_seen": 379715584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469307923771314, + "loss": 2.7782, + "theoretical_loss": 4.035400234712732, + "tokens_seen": 379781120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469207622868606, + "loss": 2.9358, + "theoretical_loss": 4.035321654718937, + "tokens_seen": 379846656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469107321965898, + "loss": 2.9011, + "theoretical_loss": 4.0352430920769855, + "tokens_seen": 379912192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044690070210631896, + "loss": 2.8626, + "theoretical_loss": 4.035164546780055, + "tokens_seen": 379977728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044689067201604814, + "loss": 2.9872, + "theoretical_loss": 4.035086018821325, + "tokens_seen": 380043264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 221040, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.943068742752075, + "objective/train/theoretical_loss": 4.035007508193978, + "objective/train/tokens_used": 400568800, + "theoretical_loss": 4.035007508193978, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468806419257773, + "loss": 3.0626, + "theoretical_loss": 4.035007508193978, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044687061183550656, + "loss": 2.7622, + "theoretical_loss": 4.034929014891202, + "tokens_seen": 380174336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468605817452357, + "loss": 2.9553, + "theoretical_loss": 4.03485053890619, + "tokens_seen": 380239872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468505516549649, + "loss": 2.7883, + "theoretical_loss": 4.034772080232137, + "tokens_seen": 380305408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044684052156469405, + "loss": 2.8372, + "theoretical_loss": 4.034693638862241, + "tokens_seen": 380370944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468304914744233, + "loss": 2.9443, + "theoretical_loss": 4.034615214789707, + "tokens_seen": 380436480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044682046138415246, + "loss": 2.7825, + "theoretical_loss": 4.034536808007741, + "tokens_seen": 380502016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044681043129388164, + "loss": 2.8113, + "theoretical_loss": 4.034458418509553, + "tokens_seen": 380567552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468004012036108, + "loss": 2.8747, + "theoretical_loss": 4.034380046288359, + "tokens_seen": 380633088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044679037111334, + "loss": 2.9584, + "theoretical_loss": 4.034301691337377, + "tokens_seen": 380698624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467803410230692, + "loss": 2.9068, + "theoretical_loss": 4.034223353649828, + "tokens_seen": 380764160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467703109327984, + "loss": 2.9041, + "theoretical_loss": 4.034145033218941, + "tokens_seen": 380829696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044676028084252755, + "loss": 2.9763, + "theoretical_loss": 4.034066730037944, + "tokens_seen": 380895232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467502507522568, + "loss": 3.2191, + "theoretical_loss": 4.0339884441000695, + "tokens_seen": 380960768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044674022066198597, + "loss": 3.015, + "theoretical_loss": 4.0339101753985585, + "tokens_seen": 381026304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044673019057171515, + "loss": 3.0623, + "theoretical_loss": 4.033831923926649, + "tokens_seen": 381091840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044672016048144433, + "loss": 2.8022, + "theoretical_loss": 4.033753689677588, + "tokens_seen": 381157376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467101303911735, + "loss": 3.0212, + "theoretical_loss": 4.033675472644624, + "tokens_seen": 381222912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467001003009027, + "loss": 2.8385, + "theoretical_loss": 4.03359727282101, + "tokens_seen": 381288448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466900702106319, + "loss": 2.9087, + "theoretical_loss": 4.033519090200002, + "tokens_seen": 381353984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466800401203611, + "loss": 2.8946, + "theoretical_loss": 4.033440924774862, + "tokens_seen": 381419520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466700100300903, + "loss": 2.7513, + "theoretical_loss": 4.033362776538853, + "tokens_seen": 381485056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044665997993981947, + "loss": 2.809, + "theoretical_loss": 4.033284645485243, + "tokens_seen": 381550592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044664994984954865, + "loss": 2.8946, + "theoretical_loss": 4.033206531607304, + "tokens_seen": 381616128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466399197592779, + "loss": 2.8353, + "theoretical_loss": 4.033128434898312, + "tokens_seen": 381681664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 221564, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5393569469451904, + "objective/train/theoretical_loss": 4.033050355351545, + "objective/train/tokens_used": 402207200, + "theoretical_loss": 4.033050355351545, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446629889669007, + "loss": 2.8139, + "theoretical_loss": 4.033050355351545, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044661985957873625, + "loss": 2.8216, + "theoretical_loss": 4.032972292960288, + "tokens_seen": 381812736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044660982948846543, + "loss": 2.8422, + "theoretical_loss": 4.0328942477178265, + "tokens_seen": 381878272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465997993981946, + "loss": 2.9652, + "theoretical_loss": 4.032816219617452, + "tokens_seen": 381943808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465897693079238, + "loss": 2.8776, + "theoretical_loss": 4.032738208652459, + "tokens_seen": 382009344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044657973921765297, + "loss": 2.8477, + "theoretical_loss": 4.032660214816145, + "tokens_seen": 382074880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044656970912738215, + "loss": 2.8979, + "theoretical_loss": 4.0325822381018135, + "tokens_seen": 382140416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465596790371114, + "loss": 2.9445, + "theoretical_loss": 4.03250427850277, + "tokens_seen": 382205952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465496489468405, + "loss": 3.0591, + "theoretical_loss": 4.032426336012321, + "tokens_seen": 382271488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044653961885656975, + "loss": 2.8794, + "theoretical_loss": 4.032348410623785, + "tokens_seen": 382337024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465295887662989, + "loss": 3.0788, + "theoretical_loss": 4.032270502330475, + "tokens_seen": 382402560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465195586760281, + "loss": 2.9419, + "theoretical_loss": 4.032192611125714, + "tokens_seen": 382468096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465095285857573, + "loss": 3.056, + "theoretical_loss": 4.032114737002825, + "tokens_seen": 382533632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464994984954865, + "loss": 3.0452, + "theoretical_loss": 4.032036879955138, + "tokens_seen": 382599168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044648946840521566, + "loss": 2.9026, + "theoretical_loss": 4.031959039975983, + "tokens_seen": 382664704 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044647943831494484, + "loss": 2.8125, + "theoretical_loss": 4.031881217058698, + "tokens_seen": 382730240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446469408224674, + "loss": 2.6746, + "theoretical_loss": 4.0318034111966226, + "tokens_seen": 382795776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044645937813440325, + "loss": 2.946, + "theoretical_loss": 4.031725622383099, + "tokens_seen": 382861312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464493480441324, + "loss": 3.0812, + "theoretical_loss": 4.031647850611473, + "tokens_seen": 382926848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464393179538616, + "loss": 3.153, + "theoretical_loss": 4.031570095875098, + "tokens_seen": 382992384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464292878635908, + "loss": 3.0693, + "theoretical_loss": 4.031492358167329, + "tokens_seen": 383057920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044641925777332, + "loss": 3.2132, + "theoretical_loss": 4.031414637481521, + "tokens_seen": 383123456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044640922768304916, + "loss": 2.6272, + "theoretical_loss": 4.031336933811038, + "tokens_seen": 383188992 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044639919759277834, + "loss": 3.0864, + "theoretical_loss": 4.031259247149245, + "tokens_seen": 383254528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463891675025075, + "loss": 3.0017, + "theoretical_loss": 4.031181577489512, + "tokens_seen": 383320064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 222655, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.689690589904785, + "objective/train/theoretical_loss": 4.031103924825214, + "objective/train/tokens_used": 403845600, + "theoretical_loss": 4.031103924825214, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044637913741223676, + "loss": 3.0431, + "theoretical_loss": 4.031103924825214, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463691073219659, + "loss": 2.8683, + "theoretical_loss": 4.0310262891497235, + "tokens_seen": 383451136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463590772316951, + "loss": 3.0141, + "theoretical_loss": 4.030948670456425, + "tokens_seen": 383516672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044634904714142425, + "loss": 2.9239, + "theoretical_loss": 4.0308710687386995, + "tokens_seen": 383582208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463390170511535, + "loss": 2.7962, + "theoretical_loss": 4.030793483989939, + "tokens_seen": 383647744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044632898696088266, + "loss": 2.9929, + "theoretical_loss": 4.0307159162035315, + "tokens_seen": 383713280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044631895687061184, + "loss": 2.9842, + "theoretical_loss": 4.030638365372874, + "tokens_seen": 383778816 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446308926780341, + "loss": 2.8692, + "theoretical_loss": 4.030560831491365, + "tokens_seen": 383844352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462988966900702, + "loss": 3.0569, + "theoretical_loss": 4.030483314552408, + "tokens_seen": 383909888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462888665997994, + "loss": 3.2353, + "theoretical_loss": 4.03040581454941, + "tokens_seen": 383975424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462788365095286, + "loss": 3.0781, + "theoretical_loss": 4.03032833147578, + "tokens_seen": 384040960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044626880641925775, + "loss": 2.7578, + "theoretical_loss": 4.030250865324932, + "tokens_seen": 384106496 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446258776328987, + "loss": 3.0714, + "theoretical_loss": 4.030173416090284, + "tokens_seen": 384172032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044624874623871617, + "loss": 2.9921, + "theoretical_loss": 4.030095983765257, + "tokens_seen": 384237568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044623871614844535, + "loss": 2.9743, + "theoretical_loss": 4.030018568343276, + "tokens_seen": 384303104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044622868605817453, + "loss": 3.0737, + "theoretical_loss": 4.02994116981777, + "tokens_seen": 384368640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462186559679037, + "loss": 2.7114, + "theoretical_loss": 4.02986378818217, + "tokens_seen": 384434176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462086258776329, + "loss": 2.8272, + "theoretical_loss": 4.029786423429913, + "tokens_seen": 384499712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461985957873621, + "loss": 3.1081, + "theoretical_loss": 4.029709075554439, + "tokens_seen": 384565248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044618856569709125, + "loss": 2.938, + "theoretical_loss": 4.02963174454919, + "tokens_seen": 384630784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461785356068205, + "loss": 2.6041, + "theoretical_loss": 4.029554430407613, + "tokens_seen": 384696320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461685055165496, + "loss": 2.8779, + "theoretical_loss": 4.0294771331231605, + "tokens_seen": 384761856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044615847542627885, + "loss": 2.8709, + "theoretical_loss": 4.029399852689284, + "tokens_seen": 384827392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044614844533600803, + "loss": 3.0417, + "theoretical_loss": 4.0293225890994435, + "tokens_seen": 384892928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461384152457372, + "loss": 2.4958, + "theoretical_loss": 4.0292453423471, + "tokens_seen": 384958464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 222910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9024524688720703, + "objective/train/theoretical_loss": 4.029168112425719, + "objective/train/tokens_used": 405484000, + "theoretical_loss": 4.029168112425719, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461283851554664, + "loss": 3.0308, + "theoretical_loss": 4.029168112425719, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044611835506519563, + "loss": 2.8817, + "theoretical_loss": 4.029090899328769, + "tokens_seen": 385089536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044610832497492476, + "loss": 2.8394, + "theoretical_loss": 4.029013703049722, + "tokens_seen": 385155072 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446098294884654, + "loss": 2.9092, + "theoretical_loss": 4.028936523582055, + "tokens_seen": 385220608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460882647943831, + "loss": 2.9076, + "theoretical_loss": 4.028859360919247, + "tokens_seen": 385286144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044607823470411235, + "loss": 3.1037, + "theoretical_loss": 4.0287822150547825, + "tokens_seen": 385351680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044606820461384153, + "loss": 2.9683, + "theoretical_loss": 4.028705085982148, + "tokens_seen": 385417216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460581745235707, + "loss": 2.8876, + "theoretical_loss": 4.028627973694833, + "tokens_seen": 385482752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460481444332999, + "loss": 2.7867, + "theoretical_loss": 4.028550878186334, + "tokens_seen": 385548288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460381143430291, + "loss": 3.0148, + "theoretical_loss": 4.028473799450149, + "tokens_seen": 385613824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044602808425275826, + "loss": 2.4792, + "theoretical_loss": 4.028396737479777, + "tokens_seen": 385679360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460180541624875, + "loss": 3.0135, + "theoretical_loss": 4.028319692268725, + "tokens_seen": 385744896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460080240722166, + "loss": 3.1208, + "theoretical_loss": 4.028242663810501, + "tokens_seen": 385810432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044599799398194586, + "loss": 2.9895, + "theoretical_loss": 4.028165652098618, + "tokens_seen": 385875968 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445987963891675, + "loss": 3.0448, + "theoretical_loss": 4.028088657126593, + "tokens_seen": 385941504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459779338014042, + "loss": 3.0796, + "theoretical_loss": 4.028011678887944, + "tokens_seen": 386007040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459679037111334, + "loss": 3.0512, + "theoretical_loss": 4.027934717376195, + "tokens_seen": 386072576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459578736208626, + "loss": 2.7118, + "theoretical_loss": 4.027857772584873, + "tokens_seen": 386138112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044594784353059176, + "loss": 3.0313, + "theoretical_loss": 4.027780844507508, + "tokens_seen": 386203648 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445937813440321, + "loss": 2.834, + "theoretical_loss": 4.0277039331376345, + "tokens_seen": 386269184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459277833500502, + "loss": 2.7997, + "theoretical_loss": 4.027627038468791, + "tokens_seen": 386334720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591775325977936, + "loss": 2.9824, + "theoretical_loss": 4.027550160494516, + "tokens_seen": 386400256 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044590772316950854, + "loss": 3.1209, + "theoretical_loss": 4.027473299208357, + "tokens_seen": 386465792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458976930792377, + "loss": 3.1398, + "theoretical_loss": 4.027396454603862, + "tokens_seen": 386531328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044588766298896696, + "loss": 2.7702, + "theoretical_loss": 4.027319626674583, + "tokens_seen": 386596864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 224330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.956303596496582, + "objective/train/theoretical_loss": 4.027242815414074, + "objective/train/tokens_used": 407122400, + "theoretical_loss": 4.027242815414074, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458776328986961, + "loss": 3.0091, + "theoretical_loss": 4.027242815414074, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458676028084253, + "loss": 3.0036, + "theoretical_loss": 4.027166020815896, + "tokens_seen": 386727936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044585757271815445, + "loss": 2.829, + "theoretical_loss": 4.027089242873612, + "tokens_seen": 386793472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458475426278837, + "loss": 3.0081, + "theoretical_loss": 4.027012481580787, + "tokens_seen": 386859008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044583751253761286, + "loss": 2.9494, + "theoretical_loss": 4.026935736930992, + "tokens_seen": 386924544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044582748244734204, + "loss": 2.8553, + "theoretical_loss": 4.0268590089178, + "tokens_seen": 386990080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458174523570712, + "loss": 2.8579, + "theoretical_loss": 4.026782297534787, + "tokens_seen": 387055616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458074222668004, + "loss": 2.9371, + "theoretical_loss": 4.026705602775536, + "tokens_seen": 387121152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457973921765296, + "loss": 2.8851, + "theoretical_loss": 4.02662892463363, + "tokens_seen": 387186688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457873620862588, + "loss": 3.1423, + "theoretical_loss": 4.026552263102657, + "tokens_seen": 387252224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044577733199598795, + "loss": 2.9309, + "theoretical_loss": 4.026475618176208, + "tokens_seen": 387317760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457673019057172, + "loss": 3.0442, + "theoretical_loss": 4.026398989847877, + "tokens_seen": 387383296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575727181544637, + "loss": 3.0007, + "theoretical_loss": 4.0263223781112645, + "tokens_seen": 387448832 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044574724172517555, + "loss": 3.0223, + "theoretical_loss": 4.026245782959971, + "tokens_seen": 387514368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044573721163490473, + "loss": 2.7477, + "theoretical_loss": 4.026169204387603, + "tokens_seen": 387579904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457271815446339, + "loss": 2.8504, + "theoretical_loss": 4.026092642387769, + "tokens_seen": 387645440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457171514543631, + "loss": 2.9407, + "theoretical_loss": 4.0260160969540815, + "tokens_seen": 387710976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457071213640923, + "loss": 3.1619, + "theoretical_loss": 4.025939568080156, + "tokens_seen": 387776512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044569709127382145, + "loss": 2.8439, + "theoretical_loss": 4.025863055759615, + "tokens_seen": 387842048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456870611835507, + "loss": 2.932, + "theoretical_loss": 4.025786559986079, + "tokens_seen": 387907584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456770310932798, + "loss": 2.8177, + "theoretical_loss": 4.025710080753175, + "tokens_seen": 387973120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044566700100300905, + "loss": 2.9714, + "theoretical_loss": 4.025633618054535, + "tokens_seen": 388038656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044565697091273823, + "loss": 3.0176, + "theoretical_loss": 4.025557171883791, + "tokens_seen": 388104192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456469408224674, + "loss": 3.023, + "theoretical_loss": 4.025480742234581, + "tokens_seen": 388169728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456369107321966, + "loss": 2.8862, + "theoretical_loss": 4.025404329100546, + "tokens_seen": 388235264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 225094, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5328869819641113, + "objective/train/theoretical_loss": 4.0253279324753315, + "objective/train/tokens_used": 408760800, + "theoretical_loss": 4.0253279324753315, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044562688064192583, + "loss": 3.2148, + "theoretical_loss": 4.0253279324753315, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044561685055165496, + "loss": 2.9028, + "theoretical_loss": 4.025251552352584, + "tokens_seen": 388366336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456068204613842, + "loss": 3.0211, + "theoretical_loss": 4.025175188725955, + "tokens_seen": 388431872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455967903711133, + "loss": 2.894, + "theoretical_loss": 4.025098841589099, + "tokens_seen": 388497408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044558676028084255, + "loss": 2.7945, + "theoretical_loss": 4.025022510935676, + "tokens_seen": 388562944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044557673019057173, + "loss": 2.944, + "theoretical_loss": 4.024946196759347, + "tokens_seen": 388628480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455667001003009, + "loss": 3.0782, + "theoretical_loss": 4.024869899053778, + "tokens_seen": 388694016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455566700100301, + "loss": 3.1571, + "theoretical_loss": 4.024793617812637, + "tokens_seen": 388759552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455466399197593, + "loss": 2.8881, + "theoretical_loss": 4.024717353029597, + "tokens_seen": 388825088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044553660982948846, + "loss": 2.9853, + "theoretical_loss": 4.024641104698335, + "tokens_seen": 388890624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455265797392177, + "loss": 2.8279, + "theoretical_loss": 4.02456487281253, + "tokens_seen": 388956160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455165496489468, + "loss": 2.8868, + "theoretical_loss": 4.024488657365863, + "tokens_seen": 389021696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044550651955867606, + "loss": 2.901, + "theoretical_loss": 4.024412458352023, + "tokens_seen": 389087232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454964894684052, + "loss": 3.0881, + "theoretical_loss": 4.024336275764698, + "tokens_seen": 389152768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454864593781344, + "loss": 2.9645, + "theoretical_loss": 4.024260109597583, + "tokens_seen": 389218304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454764292878636, + "loss": 2.8748, + "theoretical_loss": 4.024183959844375, + "tokens_seen": 389283840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454663991975928, + "loss": 2.8245, + "theoretical_loss": 4.024107826498772, + "tokens_seen": 389349376 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044545636910732196, + "loss": 2.966, + "theoretical_loss": 4.024031709554482, + "tokens_seen": 389414912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454463390170512, + "loss": 2.7041, + "theoretical_loss": 4.023955609005208, + "tokens_seen": 389480448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454363089267803, + "loss": 2.8466, + "theoretical_loss": 4.023879524844663, + "tokens_seen": 389545984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044542627883650956, + "loss": 3.0461, + "theoretical_loss": 4.023803457066561, + "tokens_seen": 389611520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454162487462387, + "loss": 3.013, + "theoretical_loss": 4.023727405664619, + "tokens_seen": 389677056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454062186559679, + "loss": 3.1216, + "theoretical_loss": 4.02365137063256, + "tokens_seen": 389742592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453961885656971, + "loss": 2.8899, + "theoretical_loss": 4.023575351964108, + "tokens_seen": 389808128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453861584754263, + "loss": 2.9788, + "theoretical_loss": 4.02349934965299, + "tokens_seen": 389873664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 226325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.121893882751465, + "objective/train/theoretical_loss": 4.023423363692939, + "objective/train/tokens_used": 410399200, + "theoretical_loss": 4.023423363692939, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044537612838515547, + "loss": 2.988, + "theoretical_loss": 4.023423363692939, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044536609829488465, + "loss": 2.8528, + "theoretical_loss": 4.023347394077689, + "tokens_seen": 390004736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044535606820461383, + "loss": 3.0006, + "theoretical_loss": 4.023271440800979, + "tokens_seen": 390070272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044534603811434306, + "loss": 2.818, + "theoretical_loss": 4.023195503856551, + "tokens_seen": 390135808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453360080240722, + "loss": 2.8456, + "theoretical_loss": 4.02311958323815, + "tokens_seen": 390201344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453259779338014, + "loss": 3.0276, + "theoretical_loss": 4.023043678939525, + "tokens_seen": 390266880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531594784353055, + "loss": 2.9954, + "theoretical_loss": 4.022967790954429, + "tokens_seen": 390332416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453059177532598, + "loss": 3.021, + "theoretical_loss": 4.022891919276617, + "tokens_seen": 390397952 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044529588766298897, + "loss": 2.8376, + "theoretical_loss": 4.022816063899848, + "tokens_seen": 390463488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044528585757271815, + "loss": 2.9268, + "theoretical_loss": 4.022740224817886, + "tokens_seen": 390529024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527582748244733, + "loss": 2.9293, + "theoretical_loss": 4.022664402024495, + "tokens_seen": 390594560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044526579739217657, + "loss": 3.2044, + "theoretical_loss": 4.0225885955134455, + "tokens_seen": 390660096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452557673019057, + "loss": 3.0436, + "theoretical_loss": 4.022512805278511, + "tokens_seen": 390725632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044524573721163493, + "loss": 2.7897, + "theoretical_loss": 4.022437031313467, + "tokens_seen": 390791168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044523570712136406, + "loss": 3.1187, + "theoretical_loss": 4.022361273612093, + "tokens_seen": 390856704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452256770310933, + "loss": 3.0885, + "theoretical_loss": 4.022285532168174, + "tokens_seen": 390922240 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044521564694082247, + "loss": 3.0223, + "theoretical_loss": 4.022209806975494, + "tokens_seen": 390987776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044520561685055165, + "loss": 2.9709, + "theoretical_loss": 4.022134098027844, + "tokens_seen": 391053312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044519558676028083, + "loss": 2.9389, + "theoretical_loss": 4.022058405319018, + "tokens_seen": 391118848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044518555667001, + "loss": 2.7071, + "theoretical_loss": 4.021982728842812, + "tokens_seen": 391184384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044517552657973925, + "loss": 2.8463, + "theoretical_loss": 4.021907068593027, + "tokens_seen": 391249920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044516549648946843, + "loss": 2.8327, + "theoretical_loss": 4.021831424563467, + "tokens_seen": 391315456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451554663991976, + "loss": 2.7659, + "theoretical_loss": 4.021755796747938, + "tokens_seen": 391380992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451454363089268, + "loss": 2.8349, + "theoretical_loss": 4.02168018514025, + "tokens_seen": 391446528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044513540621865603, + "loss": 2.9836, + "theoretical_loss": 4.021604589734219, + "tokens_seen": 391512064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 226938, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0995724201202393, + "objective/train/theoretical_loss": 4.021529010523659, + "objective/train/tokens_used": 412037600, + "theoretical_loss": 4.021529010523659, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044512537612838516, + "loss": 3.0031, + "theoretical_loss": 4.021529010523659, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451153460381144, + "loss": 2.8716, + "theoretical_loss": 4.0214534475023935, + "tokens_seen": 391643136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451053159478435, + "loss": 2.902, + "theoretical_loss": 4.0213779006642465, + "tokens_seen": 391708672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044509528585757275, + "loss": 2.9234, + "theoretical_loss": 4.021302370003044, + "tokens_seen": 391774208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044508525576730194, + "loss": 2.9735, + "theoretical_loss": 4.021226855512616, + "tokens_seen": 391839744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450752256770311, + "loss": 2.805, + "theoretical_loss": 4.021151357186799, + "tokens_seen": 391905280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450651955867603, + "loss": 2.8537, + "theoretical_loss": 4.021075875019429, + "tokens_seen": 391970816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450551654964895, + "loss": 2.779, + "theoretical_loss": 4.021000409004348, + "tokens_seen": 392036352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044504513540621866, + "loss": 2.9278, + "theoretical_loss": 4.0209249591353995, + "tokens_seen": 392101888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450351053159479, + "loss": 2.9617, + "theoretical_loss": 4.020849525406431, + "tokens_seen": 392167424 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445025075225677, + "loss": 2.8302, + "theoretical_loss": 4.020774107811294, + "tokens_seen": 392232960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044501504513540626, + "loss": 2.8883, + "theoretical_loss": 4.020698706343844, + "tokens_seen": 392298496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450050150451354, + "loss": 2.8845, + "theoretical_loss": 4.020623320997937, + "tokens_seen": 392364032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449949849548646, + "loss": 2.9504, + "theoretical_loss": 4.020547951767435, + "tokens_seen": 392429568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449849548645938, + "loss": 2.6795, + "theoretical_loss": 4.020472598646203, + "tokens_seen": 392495104 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444974924774323, + "loss": 2.8708, + "theoretical_loss": 4.020397261628108, + "tokens_seen": 392560640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044496489468405216, + "loss": 2.8069, + "theoretical_loss": 4.020321940707021, + "tokens_seen": 392626176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449548645937814, + "loss": 2.8997, + "theoretical_loss": 4.020246635876817, + "tokens_seen": 392691712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449448345035105, + "loss": 2.9409, + "theoretical_loss": 4.020171347131376, + "tokens_seen": 392757248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044493480441323976, + "loss": 2.8469, + "theoretical_loss": 4.0200960744645755, + "tokens_seen": 392822784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449247743229689, + "loss": 2.9276, + "theoretical_loss": 4.020020817870303, + "tokens_seen": 392888320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449147442326981, + "loss": 3.0255, + "theoretical_loss": 4.019945577342445, + "tokens_seen": 392953856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449047141424273, + "loss": 2.949, + "theoretical_loss": 4.019870352874894, + "tokens_seen": 393019392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448946840521565, + "loss": 2.9662, + "theoretical_loss": 4.019795144461543, + "tokens_seen": 393084928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044488465396188567, + "loss": 2.9866, + "theoretical_loss": 4.019719952096294, + "tokens_seen": 393150464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 228118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.819366931915283, + "objective/train/theoretical_loss": 4.019644775773043, + "objective/train/tokens_used": 413676000, + "theoretical_loss": 4.019644775773043, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044487462387161485, + "loss": 2.8016, + "theoretical_loss": 4.019644775773043, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044486459378134403, + "loss": 3.015, + "theoretical_loss": 4.019569615485698, + "tokens_seen": 393281536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044485456369107326, + "loss": 2.7138, + "theoretical_loss": 4.019494471228166, + "tokens_seen": 393347072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448445336008024, + "loss": 3.0278, + "theoretical_loss": 4.019419342994358, + "tokens_seen": 393412608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448345035105316, + "loss": 2.8665, + "theoretical_loss": 4.01934423077819, + "tokens_seen": 393478144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044482447342026075, + "loss": 3.0124, + "theoretical_loss": 4.019269134573579, + "tokens_seen": 393543680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044481444332999, + "loss": 2.9158, + "theoretical_loss": 4.019194054374447, + "tokens_seen": 393609216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044480441323971917, + "loss": 3.0232, + "theoretical_loss": 4.019118990174718, + "tokens_seen": 393674752 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044479438314944835, + "loss": 2.8318, + "theoretical_loss": 4.019043941968319, + "tokens_seen": 393740288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044478435305917753, + "loss": 2.8996, + "theoretical_loss": 4.018968909749184, + "tokens_seen": 393805824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044477432296890677, + "loss": 2.8596, + "theoretical_loss": 4.018893893511246, + "tokens_seen": 393871360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447642928786359, + "loss": 3.0738, + "theoretical_loss": 4.018818893248442, + "tokens_seen": 393936896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044475426278836513, + "loss": 2.8301, + "theoretical_loss": 4.018743908954715, + "tokens_seen": 394002432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044474423269809426, + "loss": 2.8219, + "theoretical_loss": 4.018668940624007, + "tokens_seen": 394067968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447342026078235, + "loss": 2.9127, + "theoretical_loss": 4.018593988250269, + "tokens_seen": 394133504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044472417251755267, + "loss": 2.9686, + "theoretical_loss": 4.01851905182745, + "tokens_seen": 394199040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044471414242728185, + "loss": 2.9248, + "theoretical_loss": 4.018444131349505, + "tokens_seen": 394264576 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044470411233701103, + "loss": 2.8442, + "theoretical_loss": 4.018369226810392, + "tokens_seen": 394330112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446940822467402, + "loss": 2.9457, + "theoretical_loss": 4.018294338204071, + "tokens_seen": 394395648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446840521564694, + "loss": 2.7568, + "theoretical_loss": 4.018219465524508, + "tokens_seen": 394461184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044467402206619863, + "loss": 2.8902, + "theoretical_loss": 4.0181446087656685, + "tokens_seen": 394526720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044466399197592776, + "loss": 2.8016, + "theoretical_loss": 4.018069767921524, + "tokens_seen": 394592256 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444653961885657, + "loss": 3.0541, + "theoretical_loss": 4.01799494298605, + "tokens_seen": 394657792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446439317953861, + "loss": 2.636, + "theoretical_loss": 4.017920133953223, + "tokens_seen": 394723328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044463390170511536, + "loss": 2.9731, + "theoretical_loss": 4.017845340817024, + "tokens_seen": 394788864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 228787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5924181938171387, + "objective/train/theoretical_loss": 4.017770563571435, + "objective/train/tokens_used": 415314400, + "theoretical_loss": 4.017770563571435, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044462387161484454, + "loss": 2.844, + "theoretical_loss": 4.017770563571435, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446138415245737, + "loss": 2.8228, + "theoretical_loss": 4.017695802210447, + "tokens_seen": 394919936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446038114343029, + "loss": 2.9139, + "theoretical_loss": 4.017621056728047, + "tokens_seen": 394985472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044459378134403214, + "loss": 3.0017, + "theoretical_loss": 4.017546327118231, + "tokens_seen": 395051008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044458375125376126, + "loss": 2.9105, + "theoretical_loss": 4.017471613374996, + "tokens_seen": 395116544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445737211634905, + "loss": 2.7957, + "theoretical_loss": 4.01739691549234, + "tokens_seen": 395182080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445636910732196, + "loss": 2.9505, + "theoretical_loss": 4.017322233464269, + "tokens_seen": 395247616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044455366098294886, + "loss": 2.8694, + "theoretical_loss": 4.017247567284789, + "tokens_seen": 395313152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044454363089267804, + "loss": 2.6045, + "theoretical_loss": 4.017172916947911, + "tokens_seen": 395378688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445336008024072, + "loss": 2.9287, + "theoretical_loss": 4.017098282447646, + "tokens_seen": 395444224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445235707121364, + "loss": 2.9332, + "theoretical_loss": 4.017023663778014, + "tokens_seen": 395509760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445135406218656, + "loss": 2.7736, + "theoretical_loss": 4.016949060933033, + "tokens_seen": 395575296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044450351053159477, + "loss": 2.9836, + "theoretical_loss": 4.016874473906726, + "tokens_seen": 395640832 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444493480441324, + "loss": 2.7708, + "theoretical_loss": 4.016799902693119, + "tokens_seen": 395706368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044448345035105313, + "loss": 2.9282, + "theoretical_loss": 4.016725347286243, + "tokens_seen": 395771904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044447342026078236, + "loss": 2.9519, + "theoretical_loss": 4.01665080768013, + "tokens_seen": 395837440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444633901705115, + "loss": 2.6818, + "theoretical_loss": 4.016576283868816, + "tokens_seen": 395902976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444533600802407, + "loss": 2.7103, + "theoretical_loss": 4.016501775846342, + "tokens_seen": 395968512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044444332998996996, + "loss": 2.9564, + "theoretical_loss": 4.016427283606747, + "tokens_seen": 396034048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444332998996991, + "loss": 2.9737, + "theoretical_loss": 4.01635280714408, + "tokens_seen": 396099584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444232698094283, + "loss": 2.8439, + "theoretical_loss": 4.016278346452389, + "tokens_seen": 396165120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444132397191575, + "loss": 2.7579, + "theoretical_loss": 4.016203901525726, + "tokens_seen": 396230656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444032096288867, + "loss": 2.9436, + "theoretical_loss": 4.0161294723581475, + "tokens_seen": 396296192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044439317953861587, + "loss": 2.9978, + "theoretical_loss": 4.016055058943712, + "tokens_seen": 396361728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044438314944834505, + "loss": 2.9448, + "theoretical_loss": 4.01598066127648, + "tokens_seen": 396427264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 229924, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7313523292541504, + "objective/train/theoretical_loss": 4.015906279350517, + "objective/train/tokens_used": 416952800, + "theoretical_loss": 4.015906279350517, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044437311935807423, + "loss": 2.895, + "theoretical_loss": 4.015906279350517, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044436308926780346, + "loss": 2.8399, + "theoretical_loss": 4.015831913159893, + "tokens_seen": 396558336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443530591775326, + "loss": 3.0352, + "theoretical_loss": 4.015757562698678, + "tokens_seen": 396623872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443430290872618, + "loss": 2.971, + "theoretical_loss": 4.015683227960948, + "tokens_seen": 396689408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044433299899699095, + "loss": 2.7352, + "theoretical_loss": 4.015608908940781, + "tokens_seen": 396754944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443229689067202, + "loss": 2.7234, + "theoretical_loss": 4.015534605632256, + "tokens_seen": 396820480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044431293881644937, + "loss": 2.9999, + "theoretical_loss": 4.01546031802946, + "tokens_seen": 396886016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044430290872617855, + "loss": 3.0871, + "theoretical_loss": 4.01538604612648, + "tokens_seen": 396951552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044429287863590773, + "loss": 2.8029, + "theoretical_loss": 4.015311789917406, + "tokens_seen": 397017088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044428284854563697, + "loss": 2.9118, + "theoretical_loss": 4.015237549396333, + "tokens_seen": 397082624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442728184553661, + "loss": 2.869, + "theoretical_loss": 4.015163324557358, + "tokens_seen": 397148160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044426278836509533, + "loss": 3.1997, + "theoretical_loss": 4.015089115394581, + "tokens_seen": 397213696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044425275827482446, + "loss": 2.8647, + "theoretical_loss": 4.015014921902106, + "tokens_seen": 397279232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442427281845537, + "loss": 2.787, + "theoretical_loss": 4.014940744074039, + "tokens_seen": 397344768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044423269809428287, + "loss": 2.9288, + "theoretical_loss": 4.014866581904491, + "tokens_seen": 397410304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044422266800401205, + "loss": 2.8941, + "theoretical_loss": 4.014792435387575, + "tokens_seen": 397475840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044421263791374123, + "loss": 3.0604, + "theoretical_loss": 4.014718304517408, + "tokens_seen": 397541376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442026078234704, + "loss": 2.9054, + "theoretical_loss": 4.0146441892881075, + "tokens_seen": 397606912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441925777331996, + "loss": 3.0049, + "theoretical_loss": 4.014570089693798, + "tokens_seen": 397672448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044418254764292883, + "loss": 3.119, + "theoretical_loss": 4.014496005728605, + "tokens_seen": 397737984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044417251755265796, + "loss": 2.8411, + "theoretical_loss": 4.014421937386658, + "tokens_seen": 397803520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441624874623872, + "loss": 3.0238, + "theoretical_loss": 4.014347884662088, + "tokens_seen": 397869056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441524573721163, + "loss": 2.7703, + "theoretical_loss": 4.014273847549032, + "tokens_seen": 397934592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044414242728184556, + "loss": 2.7537, + "theoretical_loss": 4.014199826041628, + "tokens_seen": 398000128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044413239719157474, + "loss": 2.898, + "theoretical_loss": 4.014125820134016, + "tokens_seen": 398065664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 231183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1082088947296143, + "objective/train/theoretical_loss": 4.014051829820343, + "objective/train/tokens_used": 418591200, + "theoretical_loss": 4.014051829820343, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441223671013039, + "loss": 3.0483, + "theoretical_loss": 4.014051829820343, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441123370110331, + "loss": 2.9731, + "theoretical_loss": 4.0139778550947565, + "tokens_seen": 398196736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044410230692076234, + "loss": 2.8605, + "theoretical_loss": 4.013903895951408, + "tokens_seen": 398262272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044409227683049146, + "loss": 2.9152, + "theoretical_loss": 4.013829952384452, + "tokens_seen": 398327808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440822467402207, + "loss": 3.0129, + "theoretical_loss": 4.013756024388045, + "tokens_seen": 398393344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440722166499498, + "loss": 2.9997, + "theoretical_loss": 4.013682111956348, + "tokens_seen": 398458880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044406218655967906, + "loss": 2.7039, + "theoretical_loss": 4.013608215083526, + "tokens_seen": 398524416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044405215646940824, + "loss": 2.8974, + "theoretical_loss": 4.0135343337637455, + "tokens_seen": 398589952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440421263791374, + "loss": 2.7087, + "theoretical_loss": 4.013460467991176, + "tokens_seen": 398655488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440320962888666, + "loss": 2.8162, + "theoretical_loss": 4.01338661775999, + "tokens_seen": 398721024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440220661985958, + "loss": 3.114, + "theoretical_loss": 4.013312783064366, + "tokens_seen": 398786560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044401203610832497, + "loss": 2.8509, + "theoretical_loss": 4.013238963898483, + "tokens_seen": 398852096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440020060180542, + "loss": 2.6743, + "theoretical_loss": 4.0131651602565235, + "tokens_seen": 398917632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044399197592778333, + "loss": 2.7401, + "theoretical_loss": 4.013091372132672, + "tokens_seen": 398983168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044398194583751256, + "loss": 3.0041, + "theoretical_loss": 4.01301759952112, + "tokens_seen": 399048704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439719157472417, + "loss": 2.8825, + "theoretical_loss": 4.012943842416057, + "tokens_seen": 399114240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439618856569709, + "loss": 2.8614, + "theoretical_loss": 4.01287010081168, + "tokens_seen": 399179776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439518555667001, + "loss": 2.7441, + "theoretical_loss": 4.012796374702187, + "tokens_seen": 399245312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439418254764293, + "loss": 2.7377, + "theoretical_loss": 4.012722664081779, + "tokens_seen": 399310848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044393179538615847, + "loss": 2.8877, + "theoretical_loss": 4.012648968944661, + "tokens_seen": 399376384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439217652958877, + "loss": 2.9744, + "theoretical_loss": 4.01257528928504, + "tokens_seen": 399441920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044391173520561683, + "loss": 2.6387, + "theoretical_loss": 4.012501625097128, + "tokens_seen": 399507456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044390170511534607, + "loss": 2.8492, + "theoretical_loss": 4.012427976375139, + "tokens_seen": 399572992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438916750250752, + "loss": 2.9283, + "theoretical_loss": 4.012354343113288, + "tokens_seen": 399638528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044388164493480443, + "loss": 2.8775, + "theoretical_loss": 4.012280725305798, + "tokens_seen": 399704064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 231706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.14528226852417, + "objective/train/theoretical_loss": 4.01220712294689, + "objective/train/tokens_used": 420229600, + "theoretical_loss": 4.01220712294689, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438716148445336, + "loss": 3.0579, + "theoretical_loss": 4.01220712294689, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438615847542628, + "loss": 2.7976, + "theoretical_loss": 4.012133536030791, + "tokens_seen": 399835136 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044385155466399197, + "loss": 2.9258, + "theoretical_loss": 4.012059964551731, + "tokens_seen": 399900672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044384152457372115, + "loss": 2.7009, + "theoretical_loss": 4.011986408503942, + "tokens_seen": 399966208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044383149448345033, + "loss": 2.7834, + "theoretical_loss": 4.0119128678816605, + "tokens_seen": 400031744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044382146439317957, + "loss": 2.9803, + "theoretical_loss": 4.011839342679125, + "tokens_seen": 400097280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438114343029087, + "loss": 2.6878, + "theoretical_loss": 4.0117658328905765, + "tokens_seen": 400162816 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044380140421263793, + "loss": 2.8165, + "theoretical_loss": 4.011692338510262, + "tokens_seen": 400228352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437913741223671, + "loss": 2.6641, + "theoretical_loss": 4.011618859532426, + "tokens_seen": 400293888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437813440320963, + "loss": 2.8163, + "theoretical_loss": 4.011545395951323, + "tokens_seen": 400359424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437713139418255, + "loss": 2.6949, + "theoretical_loss": 4.011471947761207, + "tokens_seen": 400424960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044376128385155466, + "loss": 3.0059, + "theoretical_loss": 4.011398514956333, + "tokens_seen": 400490496 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044375125376128384, + "loss": 2.8145, + "theoretical_loss": 4.011325097530963, + "tokens_seen": 400556032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044374122367101307, + "loss": 2.6344, + "theoretical_loss": 4.011251695479361, + "tokens_seen": 400621568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437311935807422, + "loss": 3.0702, + "theoretical_loss": 4.011178308795792, + "tokens_seen": 400687104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044372116349047143, + "loss": 2.8061, + "theoretical_loss": 4.011104937474527, + "tokens_seen": 400752640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044371113340020056, + "loss": 2.6934, + "theoretical_loss": 4.011031581509838, + "tokens_seen": 400818176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437011033099298, + "loss": 2.8448, + "theoretical_loss": 4.0109582408960005, + "tokens_seen": 400883712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044369107321965903, + "loss": 3.0339, + "theoretical_loss": 4.010884915627294, + "tokens_seen": 400949248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044368104312938816, + "loss": 2.77, + "theoretical_loss": 4.0108116056980005, + "tokens_seen": 401014784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436710130391174, + "loss": 2.9177, + "theoretical_loss": 4.010738311102405, + "tokens_seen": 401080320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436609829488465, + "loss": 2.8799, + "theoretical_loss": 4.010665031834796, + "tokens_seen": 401145856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044365095285857576, + "loss": 2.6816, + "theoretical_loss": 4.010591767889462, + "tokens_seen": 401211392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044364092276830494, + "loss": 2.627, + "theoretical_loss": 4.0105185192607, + "tokens_seen": 401276928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436308926780341, + "loss": 2.8835, + "theoretical_loss": 4.010445285942806, + "tokens_seen": 401342464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 232929, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.049757242202759, + "objective/train/theoretical_loss": 4.0103720679300805, + "objective/train/tokens_used": 421868000, + "theoretical_loss": 4.0103720679300805, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436208625877633, + "loss": 2.743, + "theoretical_loss": 4.0103720679300805, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044361083249749254, + "loss": 2.7237, + "theoretical_loss": 4.010298865216827, + "tokens_seen": 401473536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044360080240722166, + "loss": 2.8752, + "theoretical_loss": 4.010225677797351, + "tokens_seen": 401539072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435907723169509, + "loss": 2.9571, + "theoretical_loss": 4.010152505665964, + "tokens_seen": 401604608 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044358074222668, + "loss": 2.823, + "theoretical_loss": 4.010079348816976, + "tokens_seen": 401670144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044357071213640926, + "loss": 2.8259, + "theoretical_loss": 4.010006207244704, + "tokens_seen": 401735680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044356068204613844, + "loss": 2.7683, + "theoretical_loss": 4.009933080943466, + "tokens_seen": 401801216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435506519558676, + "loss": 3.0118, + "theoretical_loss": 4.009859969907584, + "tokens_seen": 401866752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435406218655968, + "loss": 2.8564, + "theoretical_loss": 4.0097868741313825, + "tokens_seen": 401932288 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443530591775326, + "loss": 2.7656, + "theoretical_loss": 4.009713793609189, + "tokens_seen": 401997824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044352056168505517, + "loss": 2.6552, + "theoretical_loss": 4.009640728335333, + "tokens_seen": 402063360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435105315947844, + "loss": 2.995, + "theoretical_loss": 4.009567678304151, + "tokens_seen": 402128896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044350050150451353, + "loss": 2.8946, + "theoretical_loss": 4.009494643509978, + "tokens_seen": 402194432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044349047141424276, + "loss": 2.6877, + "theoretical_loss": 4.009421623947153, + "tokens_seen": 402259968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434804413239719, + "loss": 2.9107, + "theoretical_loss": 4.00934861961002, + "tokens_seen": 402325504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434704112337011, + "loss": 3.044, + "theoretical_loss": 4.009275630492926, + "tokens_seen": 402391040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434603811434303, + "loss": 2.9364, + "theoretical_loss": 4.009202656590217, + "tokens_seen": 402456576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434503510531595, + "loss": 2.9931, + "theoretical_loss": 4.009129697896247, + "tokens_seen": 402522112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044344032096288867, + "loss": 2.7369, + "theoretical_loss": 4.00905675440537, + "tokens_seen": 402587648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434302908726179, + "loss": 2.7316, + "theoretical_loss": 4.008983826111945, + "tokens_seen": 402653184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044342026078234703, + "loss": 2.8793, + "theoretical_loss": 4.008910913010331, + "tokens_seen": 402718720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044341023069207627, + "loss": 2.7036, + "theoretical_loss": 4.008838015094893, + "tokens_seen": 402784256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434002006018054, + "loss": 2.7359, + "theoretical_loss": 4.008765132359999, + "tokens_seen": 402849792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044339017051153463, + "loss": 2.8239, + "theoretical_loss": 4.0086922648000165, + "tokens_seen": 402915328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433801404212638, + "loss": 3.0154, + "theoretical_loss": 4.00861941240932, + "tokens_seen": 402980864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 233431, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7376959323883057, + "objective/train/theoretical_loss": 4.008546575182286, + "objective/train/tokens_used": 423506400, + "theoretical_loss": 4.008546575182286, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443370110330993, + "loss": 2.7459, + "theoretical_loss": 4.008546575182286, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044336008024072217, + "loss": 2.8226, + "theoretical_loss": 4.008473753113292, + "tokens_seen": 403111936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044335005015045135, + "loss": 2.8344, + "theoretical_loss": 4.008400946196721, + "tokens_seen": 403177472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044334002006018053, + "loss": 3.1402, + "theoretical_loss": 4.008328154426956, + "tokens_seen": 403243008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044332998996990977, + "loss": 2.8666, + "theoretical_loss": 4.008255377798388, + "tokens_seen": 403308544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433199598796389, + "loss": 2.8195, + "theoretical_loss": 4.008182616305406, + "tokens_seen": 403374080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044330992978936813, + "loss": 2.707, + "theoretical_loss": 4.008109869942404, + "tokens_seen": 403439616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432998996990973, + "loss": 2.7543, + "theoretical_loss": 4.008037138703779, + "tokens_seen": 403505152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432898696088265, + "loss": 3.1115, + "theoretical_loss": 4.007964422583931, + "tokens_seen": 403570688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432798395185557, + "loss": 2.8065, + "theoretical_loss": 4.007891721577263, + "tokens_seen": 403636224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044326980942828486, + "loss": 2.8883, + "theoretical_loss": 4.00781903567818, + "tokens_seen": 403701760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044325977933801404, + "loss": 2.7961, + "theoretical_loss": 4.0077463648810925, + "tokens_seen": 403767296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044324974924774327, + "loss": 3.0498, + "theoretical_loss": 4.007673709180411, + "tokens_seen": 403832832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432397191574724, + "loss": 2.8936, + "theoretical_loss": 4.007601068570549, + "tokens_seen": 403898368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044322968906720163, + "loss": 2.7508, + "theoretical_loss": 4.007528443045928, + "tokens_seen": 403963904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044321965897693076, + "loss": 3.0503, + "theoretical_loss": 4.007455832600965, + "tokens_seen": 404029440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044320962888666, + "loss": 3.0522, + "theoretical_loss": 4.007383237230086, + "tokens_seen": 404094976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431995987963892, + "loss": 2.5872, + "theoretical_loss": 4.007310656927716, + "tokens_seen": 404160512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044318956870611836, + "loss": 2.891, + "theoretical_loss": 4.0072380916882855, + "tokens_seen": 404226048 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044317953861584754, + "loss": 3.0283, + "theoretical_loss": 4.007165541506227, + "tokens_seen": 404291584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431695085255767, + "loss": 2.9584, + "theoretical_loss": 4.007093006375976, + "tokens_seen": 404357120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431594784353059, + "loss": 2.9348, + "theoretical_loss": 4.007020486291972, + "tokens_seen": 404422656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044314944834503514, + "loss": 2.949, + "theoretical_loss": 4.006947981248654, + "tokens_seen": 404488192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044313941825476427, + "loss": 2.7082, + "theoretical_loss": 4.006875491240468, + "tokens_seen": 404553728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431293881644935, + "loss": 2.7423, + "theoretical_loss": 4.006803016261861, + "tokens_seen": 404619264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 234514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.404675245285034, + "objective/train/theoretical_loss": 4.006730556307284, + "objective/train/tokens_used": 425144800, + "theoretical_loss": 4.006730556307284, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431193580742227, + "loss": 3.1039, + "theoretical_loss": 4.006730556307284, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044310932798395186, + "loss": 2.6791, + "theoretical_loss": 4.006658111371189, + "tokens_seen": 404750336 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044309929789368104, + "loss": 2.6862, + "theoretical_loss": 4.006585681448033, + "tokens_seen": 404815872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430892678034102, + "loss": 2.8161, + "theoretical_loss": 4.006513266532275, + "tokens_seen": 404881408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430792377131394, + "loss": 2.8454, + "theoretical_loss": 4.006440866618378, + "tokens_seen": 404946944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044306920762286864, + "loss": 2.9207, + "theoretical_loss": 4.006368481700806, + "tokens_seen": 405012480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044305917753259777, + "loss": 2.7304, + "theoretical_loss": 4.0062961117740254, + "tokens_seen": 405078016 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443049147442327, + "loss": 2.8102, + "theoretical_loss": 4.00622375683251, + "tokens_seen": 405143552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044303911735205613, + "loss": 2.7443, + "theoretical_loss": 4.006151416870731, + "tokens_seen": 405209088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044302908726178537, + "loss": 2.8585, + "theoretical_loss": 4.0060790918831675, + "tokens_seen": 405274624 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044301905717151455, + "loss": 2.832, + "theoretical_loss": 4.006006781864298, + "tokens_seen": 405340160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044300902708124373, + "loss": 2.8704, + "theoretical_loss": 4.005934486808605, + "tokens_seen": 405405696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429989969909729, + "loss": 2.8721, + "theoretical_loss": 4.0058622067105745, + "tokens_seen": 405471232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429889669007021, + "loss": 3.0217, + "theoretical_loss": 4.005789941564695, + "tokens_seen": 405536768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044297893681043127, + "loss": 2.781, + "theoretical_loss": 4.005717691365457, + "tokens_seen": 405602304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429689067201605, + "loss": 2.7174, + "theoretical_loss": 4.005645456107356, + "tokens_seen": 405667840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044295887662988963, + "loss": 2.5529, + "theoretical_loss": 4.005573235784888, + "tokens_seen": 405733376 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294884653961887, + "loss": 2.9012, + "theoretical_loss": 4.005501030392553, + "tokens_seen": 405798912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429388164493481, + "loss": 3.0316, + "theoretical_loss": 4.005428839924855, + "tokens_seen": 405864448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044292878635907723, + "loss": 2.766, + "theoretical_loss": 4.0053566643763014, + "tokens_seen": 405929984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044291875626880647, + "loss": 2.9476, + "theoretical_loss": 4.0052845037413976, + "tokens_seen": 405995520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429087261785356, + "loss": 3.0683, + "theoretical_loss": 4.005212358014658, + "tokens_seen": 406061056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044289869608826483, + "loss": 2.8003, + "theoretical_loss": 4.005140227190596, + "tokens_seen": 406126592 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442888665997994, + "loss": 2.7452, + "theoretical_loss": 4.00506811126373, + "tokens_seen": 406192128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428786359077232, + "loss": 2.8949, + "theoretical_loss": 4.00499601022858, + "tokens_seen": 406257664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 235119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8222172260284424, + "objective/train/theoretical_loss": 4.0049239240796695, + "objective/train/tokens_used": 426783200, + "theoretical_loss": 4.0049239240796695, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044286860581745237, + "loss": 2.999, + "theoretical_loss": 4.0049239240796695, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044285857572718155, + "loss": 2.8565, + "theoretical_loss": 4.004851852811523, + "tokens_seen": 406388736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044284854563691073, + "loss": 2.8309, + "theoretical_loss": 4.004779796418673, + "tokens_seen": 406454272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044283851554663997, + "loss": 2.713, + "theoretical_loss": 4.00470775489565, + "tokens_seen": 406519808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428284854563691, + "loss": 2.9481, + "theoretical_loss": 4.004635728236988, + "tokens_seen": 406585344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044281845536609833, + "loss": 2.8521, + "theoretical_loss": 4.004563716437227, + "tokens_seen": 406650880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428084252758275, + "loss": 2.8142, + "theoretical_loss": 4.004491719490905, + "tokens_seen": 406716416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427983951855567, + "loss": 2.6657, + "theoretical_loss": 4.004419737392568, + "tokens_seen": 406781952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427883650952859, + "loss": 2.8134, + "theoretical_loss": 4.004347770136761, + "tokens_seen": 406847488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044277833500501506, + "loss": 2.6711, + "theoretical_loss": 4.0042758177180335, + "tokens_seen": 406913024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044276830491474424, + "loss": 2.8276, + "theoretical_loss": 4.004203880130939, + "tokens_seen": 406978560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044275827482447347, + "loss": 2.881, + "theoretical_loss": 4.004131957370031, + "tokens_seen": 407044096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427482447342026, + "loss": 2.9221, + "theoretical_loss": 4.004060049429869, + "tokens_seen": 407109632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044273821464393184, + "loss": 2.7697, + "theoretical_loss": 4.003988156305013, + "tokens_seen": 407175168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044272818455366096, + "loss": 2.764, + "theoretical_loss": 4.003916277990028, + "tokens_seen": 407240704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427181544633902, + "loss": 2.8386, + "theoretical_loss": 4.003844414479477, + "tokens_seen": 407306240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427081243731194, + "loss": 2.9933, + "theoretical_loss": 4.003772565767934, + "tokens_seen": 407371776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044269809428284856, + "loss": 2.7289, + "theoretical_loss": 4.003700731849969, + "tokens_seen": 407437312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044268806419257774, + "loss": 2.9191, + "theoretical_loss": 4.003628912720157, + "tokens_seen": 407502848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426780341023069, + "loss": 2.6767, + "theoretical_loss": 4.003557108373077, + "tokens_seen": 407568384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426680040120361, + "loss": 2.907, + "theoretical_loss": 4.00348531880331, + "tokens_seen": 407633920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044265797392176534, + "loss": 2.7716, + "theoretical_loss": 4.003413544005439, + "tokens_seen": 407699456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044264794383149447, + "loss": 2.8966, + "theoretical_loss": 4.003341783974051, + "tokens_seen": 407764992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426379137412237, + "loss": 2.8671, + "theoretical_loss": 4.003270038703736, + "tokens_seen": 407830528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426278836509529, + "loss": 2.6621, + "theoretical_loss": 4.003198308189086, + "tokens_seen": 407896064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 236101, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.368303060531616, + "objective/train/theoretical_loss": 4.003126592424696, + "objective/train/tokens_used": 428421600, + "theoretical_loss": 4.003126592424696, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044261785356068206, + "loss": 2.6108, + "theoretical_loss": 4.003126592424696, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044260782347041124, + "loss": 2.7856, + "theoretical_loss": 4.003054891405164, + "tokens_seen": 408027136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425977933801404, + "loss": 2.846, + "theoretical_loss": 4.002983205125091, + "tokens_seen": 408092672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425877632898696, + "loss": 2.5993, + "theoretical_loss": 4.00291153357908, + "tokens_seen": 408158208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044257773319959884, + "loss": 2.9444, + "theoretical_loss": 4.002839876761739, + "tokens_seen": 408223744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044256770310932797, + "loss": 2.8485, + "theoretical_loss": 4.002768234667677, + "tokens_seen": 408289280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425576730190572, + "loss": 2.812, + "theoretical_loss": 4.002696607291506, + "tokens_seen": 408354816 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044254764292878633, + "loss": 2.9355, + "theoretical_loss": 4.0026249946278405, + "tokens_seen": 408420352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044253761283851557, + "loss": 2.7608, + "theoretical_loss": 4.0025533966713, + "tokens_seen": 408485888 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044252758274824475, + "loss": 2.7582, + "theoretical_loss": 4.002481813416504, + "tokens_seen": 408551424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044251755265797393, + "loss": 2.8645, + "theoretical_loss": 4.002410244858076, + "tokens_seen": 408616960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425075225677031, + "loss": 2.9369, + "theoretical_loss": 4.002338690990644, + "tokens_seen": 408682496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424974924774323, + "loss": 2.7085, + "theoretical_loss": 4.0022671518088355, + "tokens_seen": 408748032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044248746238716147, + "loss": 2.7926, + "theoretical_loss": 4.002195627307283, + "tokens_seen": 408813568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424774322968907, + "loss": 2.7285, + "theoretical_loss": 4.002124117480621, + "tokens_seen": 408879104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246740220661983, + "loss": 2.8286, + "theoretical_loss": 4.002052622323489, + "tokens_seen": 408944640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044245737211634907, + "loss": 3.0013, + "theoretical_loss": 4.001981141830525, + "tokens_seen": 409010176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044244734202607825, + "loss": 2.5402, + "theoretical_loss": 4.001909675996375, + "tokens_seen": 409075712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044243731193580743, + "loss": 2.7802, + "theoretical_loss": 4.001838224815683, + "tokens_seen": 409141248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424272818455366, + "loss": 2.6874, + "theoretical_loss": 4.001766788283098, + "tokens_seen": 409206784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424172517552658, + "loss": 2.8716, + "theoretical_loss": 4.001695366393275, + "tokens_seen": 409272320 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442407221664995, + "loss": 2.636, + "theoretical_loss": 4.001623959140864, + "tokens_seen": 409337856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423971915747242, + "loss": 3.065, + "theoretical_loss": 4.001552566520525, + "tokens_seen": 409403392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044238716148445334, + "loss": 2.9232, + "theoretical_loss": 4.001481188526918, + "tokens_seen": 409468928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044237713139418257, + "loss": 2.7951, + "theoretical_loss": 4.001409825154705, + "tokens_seen": 409534464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 237324, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1303606033325195, + "objective/train/theoretical_loss": 4.001338476398553, + "objective/train/tokens_used": 430060000, + "theoretical_loss": 4.001338476398553, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423671013039117, + "loss": 3.0079, + "theoretical_loss": 4.001338476398553, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044235707121364093, + "loss": 2.9637, + "theoretical_loss": 4.0012671422531305, + "tokens_seen": 409665536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423470411233701, + "loss": 2.6292, + "theoretical_loss": 4.001195822713108, + "tokens_seen": 409731072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423370110330993, + "loss": 2.883, + "theoretical_loss": 4.001124517773161, + "tokens_seen": 409796608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423269809428285, + "loss": 2.99, + "theoretical_loss": 4.001053227427966, + "tokens_seen": 409862144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423169508525577, + "loss": 2.7892, + "theoretical_loss": 4.000981951672202, + "tokens_seen": 409927680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230692076228684, + "loss": 2.9052, + "theoretical_loss": 4.000910690500552, + "tokens_seen": 409993216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422968906720161, + "loss": 3.0222, + "theoretical_loss": 4.000839443907702, + "tokens_seen": 410058752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422868605817452, + "loss": 2.8375, + "theoretical_loss": 4.000768211888339, + "tokens_seen": 410124288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044227683049147444, + "loss": 2.6957, + "theoretical_loss": 4.000696994437154, + "tokens_seen": 410189824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422668004012036, + "loss": 2.7352, + "theoretical_loss": 4.000625791548843, + "tokens_seen": 410255360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422567703109328, + "loss": 2.7709, + "theoretical_loss": 4.0005546032181, + "tokens_seen": 410320896 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442246740220662, + "loss": 2.9533, + "theoretical_loss": 4.000483429439624, + "tokens_seen": 410386432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044223671013039116, + "loss": 2.874, + "theoretical_loss": 4.00041227020812, + "tokens_seen": 410451968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044222668004012034, + "loss": 3.1148, + "theoretical_loss": 4.00034112551829, + "tokens_seen": 410517504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422166499498496, + "loss": 3.0062, + "theoretical_loss": 4.000269995364842, + "tokens_seen": 410583040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422066198595787, + "loss": 2.9177, + "theoretical_loss": 4.000198879742488, + "tokens_seen": 410648576 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044219658976930794, + "loss": 2.7881, + "theoretical_loss": 4.000127778645939, + "tokens_seen": 410714112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421865596790371, + "loss": 2.7334, + "theoretical_loss": 4.000056692069911, + "tokens_seen": 410779648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421765295887663, + "loss": 2.8035, + "theoretical_loss": 3.9999856200091255, + "tokens_seen": 410845184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044216649949849554, + "loss": 2.8879, + "theoretical_loss": 3.9999145624583017, + "tokens_seen": 410910720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044215646940822467, + "loss": 2.8508, + "theoretical_loss": 3.9998435194121633, + "tokens_seen": 410976256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421464393179539, + "loss": 2.8841, + "theoretical_loss": 3.999772490865439, + "tokens_seen": 411041792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421364092276831, + "loss": 2.7457, + "theoretical_loss": 3.9997014768128567, + "tokens_seen": 411107328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044212637913741226, + "loss": 3.0086, + "theoretical_loss": 3.9996304772491498, + "tokens_seen": 411172864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 237892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8793866634368896, + "objective/train/theoretical_loss": 3.9995594921690536, + "objective/train/tokens_used": 431698400, + "theoretical_loss": 3.9995594921690536, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044211634904714144, + "loss": 2.8024, + "theoretical_loss": 3.9995594921690536, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421063189568706, + "loss": 2.7261, + "theoretical_loss": 3.9994885215673057, + "tokens_seen": 411303936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420962888665998, + "loss": 2.9451, + "theoretical_loss": 3.9994175654386472, + "tokens_seen": 411369472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044208625877632904, + "loss": 2.7401, + "theoretical_loss": 3.9993466237778215, + "tokens_seen": 411435008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044207622868605817, + "loss": 2.9415, + "theoretical_loss": 3.9992756965795744, + "tokens_seen": 411500544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420661985957874, + "loss": 2.8241, + "theoretical_loss": 3.999204783838655, + "tokens_seen": 411566080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044205616850551653, + "loss": 2.9024, + "theoretical_loss": 3.9991338855498153, + "tokens_seen": 411631616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044204613841524577, + "loss": 2.9414, + "theoretical_loss": 3.9990630017078095, + "tokens_seen": 411697152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044203610832497495, + "loss": 2.8594, + "theoretical_loss": 3.9989921323073947, + "tokens_seen": 411762688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044202607823470413, + "loss": 2.9609, + "theoretical_loss": 3.998921277343331, + "tokens_seen": 411828224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420160481444333, + "loss": 3.0881, + "theoretical_loss": 3.9988504368103817, + "tokens_seen": 411893760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420060180541625, + "loss": 2.573, + "theoretical_loss": 3.9987796107033104, + "tokens_seen": 411959296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044199598796389167, + "loss": 2.9766, + "theoretical_loss": 3.9987087990168866, + "tokens_seen": 412024832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419859578736209, + "loss": 2.9167, + "theoretical_loss": 3.998638001745881, + "tokens_seen": 412090368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044197592778335003, + "loss": 2.6952, + "theoretical_loss": 3.998567218885067, + "tokens_seen": 412155904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044196589769307927, + "loss": 2.8777, + "theoretical_loss": 3.9984964504292204, + "tokens_seen": 412221440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044195586760280845, + "loss": 2.8639, + "theoretical_loss": 3.998425696373121, + "tokens_seen": 412286976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044194583751253763, + "loss": 2.8543, + "theoretical_loss": 3.998354956711551, + "tokens_seen": 412352512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419358074222668, + "loss": 3.0178, + "theoretical_loss": 3.9982842314392935, + "tokens_seen": 412418048 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441925777331996, + "loss": 2.7787, + "theoretical_loss": 3.9982135205511358, + "tokens_seen": 412483584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419157472417252, + "loss": 3.1016, + "theoretical_loss": 3.9981428240418686, + "tokens_seen": 412549120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419057171514544, + "loss": 2.6455, + "theoretical_loss": 3.998072141906284, + "tokens_seen": 412614656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044189568706118354, + "loss": 2.9787, + "theoretical_loss": 3.998001474139178, + "tokens_seen": 412680192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044188565697091277, + "loss": 2.9904, + "theoretical_loss": 3.997930820735348, + "tokens_seen": 412745728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418756268806419, + "loss": 2.9195, + "theoretical_loss": 3.9978601816895942, + "tokens_seen": 412811264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 239244, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.011247396469116, + "objective/train/theoretical_loss": 3.9977895569967217, + "objective/train/tokens_used": 433336800, + "theoretical_loss": 3.9977895569967217, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044186559679037113, + "loss": 2.8852, + "theoretical_loss": 3.9977895569967217, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418555667001003, + "loss": 2.8744, + "theoretical_loss": 3.997718946651535, + "tokens_seen": 412942336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418455366098295, + "loss": 2.5262, + "theoretical_loss": 3.997648350648843, + "tokens_seen": 413007872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418355065195587, + "loss": 2.8229, + "theoretical_loss": 3.997577768983459, + "tokens_seen": 413073408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418254764292879, + "loss": 2.7307, + "theoretical_loss": 3.997507201650196, + "tokens_seen": 413138944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044181544633901704, + "loss": 2.87, + "theoretical_loss": 3.9974366486438706, + "tokens_seen": 413204480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418054162487463, + "loss": 2.7706, + "theoretical_loss": 3.9973661099593034, + "tokens_seen": 413270016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417953861584754, + "loss": 2.9375, + "theoretical_loss": 3.9972955855913157, + "tokens_seen": 413335552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044178535606820464, + "loss": 2.6413, + "theoretical_loss": 3.9972250755347334, + "tokens_seen": 413401088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417753259779338, + "loss": 2.741, + "theoretical_loss": 3.997154579784384, + "tokens_seen": 413466624 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441765295887663, + "loss": 2.8083, + "theoretical_loss": 3.997084098335098, + "tokens_seen": 413532160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417552657973922, + "loss": 2.9568, + "theoretical_loss": 3.997013631181708, + "tokens_seen": 413597696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044174523570712136, + "loss": 2.906, + "theoretical_loss": 3.99694317831905, + "tokens_seen": 413663232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044173520561685054, + "loss": 2.9089, + "theoretical_loss": 3.9968727397419634, + "tokens_seen": 413728768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417251755265798, + "loss": 3.0527, + "theoretical_loss": 3.9968023154452883, + "tokens_seen": 413794304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417151454363089, + "loss": 2.7741, + "theoretical_loss": 3.9967319054238684, + "tokens_seen": 413859840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044170511534603814, + "loss": 2.9615, + "theoretical_loss": 3.996661509672551, + "tokens_seen": 413925376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044169508525576727, + "loss": 3.0461, + "theoretical_loss": 3.996591128186185, + "tokens_seen": 413990912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416850551654965, + "loss": 2.776, + "theoretical_loss": 3.996520760959622, + "tokens_seen": 414056448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416750250752257, + "loss": 2.8094, + "theoretical_loss": 3.996450407987716, + "tokens_seen": 414121984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044166499498495487, + "loss": 2.9947, + "theoretical_loss": 3.9963800692653253, + "tokens_seen": 414187520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044165496489468405, + "loss": 2.7043, + "theoretical_loss": 3.9963097447873097, + "tokens_seen": 414253056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416449348044133, + "loss": 2.8336, + "theoretical_loss": 3.9962394345485315, + "tokens_seen": 414318592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416349047141424, + "loss": 2.6923, + "theoretical_loss": 3.9961691385438556, + "tokens_seen": 414384128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044162487462387164, + "loss": 2.8808, + "theoretical_loss": 3.99609885676815, + "tokens_seen": 414449664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 239946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0474255084991455, + "objective/train/theoretical_loss": 3.9960285892162855, + "objective/train/tokens_used": 434975200, + "theoretical_loss": 3.9960285892162855, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044161484453360077, + "loss": 2.9599, + "theoretical_loss": 3.9960285892162855, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044160481444333, + "loss": 2.5837, + "theoretical_loss": 3.995958335883135, + "tokens_seen": 414580736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415947843530592, + "loss": 2.6826, + "theoretical_loss": 3.995888096763574, + "tokens_seen": 414646272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044158475426278837, + "loss": 2.7732, + "theoretical_loss": 3.9958178718524824, + "tokens_seen": 414711808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044157472417251755, + "loss": 2.8478, + "theoretical_loss": 3.9957476611447396, + "tokens_seen": 414777344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044156469408224673, + "loss": 2.7315, + "theoretical_loss": 3.9956774646352304, + "tokens_seen": 414842880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415546639919759, + "loss": 2.796, + "theoretical_loss": 3.9956072823188418, + "tokens_seen": 414908416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044154463390170515, + "loss": 2.9685, + "theoretical_loss": 3.9955371141904616, + "tokens_seen": 414973952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415346038114343, + "loss": 2.9153, + "theoretical_loss": 3.995466960244982, + "tokens_seen": 415039488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415245737211635, + "loss": 2.7571, + "theoretical_loss": 3.995396820477298, + "tokens_seen": 415105024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044151454363089264, + "loss": 2.8539, + "theoretical_loss": 3.995326694882306, + "tokens_seen": 415170560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044150451354062187, + "loss": 2.9172, + "theoretical_loss": 3.9952565834549056, + "tokens_seen": 415236096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044149448345035105, + "loss": 2.9305, + "theoretical_loss": 3.9951864861899997, + "tokens_seen": 415301632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044148445336008023, + "loss": 2.9859, + "theoretical_loss": 3.995116403082493, + "tokens_seen": 415367168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414744232698094, + "loss": 2.755, + "theoretical_loss": 3.995046334127293, + "tokens_seen": 415432704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044146439317953865, + "loss": 2.809, + "theoretical_loss": 3.99497627931931, + "tokens_seen": 415498240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414543630892678, + "loss": 2.9747, + "theoretical_loss": 3.994906238653457, + "tokens_seen": 415563776 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441444332998997, + "loss": 2.8364, + "theoretical_loss": 3.9948362121246497, + "tokens_seen": 415629312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414343029087262, + "loss": 2.613, + "theoretical_loss": 3.9947661997278052, + "tokens_seen": 415694848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414242728184554, + "loss": 2.5021, + "theoretical_loss": 3.9946962014578453, + "tokens_seen": 415760384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414142427281846, + "loss": 2.8959, + "theoretical_loss": 3.994626217309693, + "tokens_seen": 415825920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044140421263791374, + "loss": 2.8188, + "theoretical_loss": 3.994556247278275, + "tokens_seen": 415891456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044139418254764297, + "loss": 2.9287, + "theoretical_loss": 3.994486291358519, + "tokens_seen": 415956992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413841524573721, + "loss": 2.9442, + "theoretical_loss": 3.9944163495453564, + "tokens_seen": 416022528 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044137412236710133, + "loss": 2.8413, + "theoretical_loss": 3.994346421833721, + "tokens_seen": 416088064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 241045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6522488594055176, + "objective/train/theoretical_loss": 3.99427650821855, + "objective/train/tokens_used": 436613600, + "theoretical_loss": 3.99427650821855, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413640922768305, + "loss": 2.9212, + "theoretical_loss": 3.99427650821855, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413540621865597, + "loss": 2.8946, + "theoretical_loss": 3.994206608694782, + "tokens_seen": 416219136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413440320962889, + "loss": 2.7445, + "theoretical_loss": 3.994136723257358, + "tokens_seen": 416284672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413340020060181, + "loss": 2.782, + "theoretical_loss": 3.994066851901224, + "tokens_seen": 416350208 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044132397191574724, + "loss": 2.8814, + "theoretical_loss": 3.993996994621326, + "tokens_seen": 416415744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413139418254765, + "loss": 2.82, + "theoretical_loss": 3.9939271514126125, + "tokens_seen": 416481280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413039117352056, + "loss": 2.7661, + "theoretical_loss": 3.9938573222700375, + "tokens_seen": 416546816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044129388164493484, + "loss": 2.9603, + "theoretical_loss": 3.9937875071885554, + "tokens_seen": 416612352 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441283851554664, + "loss": 3.1666, + "theoretical_loss": 3.993717706163122, + "tokens_seen": 416677888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412738214643932, + "loss": 2.768, + "theoretical_loss": 3.9936479191886995, + "tokens_seen": 416743424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412637913741224, + "loss": 2.9516, + "theoretical_loss": 3.9935781462602487, + "tokens_seen": 416808960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044125376128385156, + "loss": 2.82, + "theoretical_loss": 3.9935083873727355, + "tokens_seen": 416874496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044124373119358074, + "loss": 2.9864, + "theoretical_loss": 3.9934386425211277, + "tokens_seen": 416940032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044123370110331, + "loss": 2.8075, + "theoretical_loss": 3.9933689117003954, + "tokens_seen": 417005568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412236710130391, + "loss": 2.7716, + "theoretical_loss": 3.9932991949055117, + "tokens_seen": 417071104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044121364092276834, + "loss": 2.6797, + "theoretical_loss": 3.993229492131452, + "tokens_seen": 417136640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044120361083249747, + "loss": 2.8761, + "theoretical_loss": 3.9931598033731945, + "tokens_seen": 417202176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411935807422267, + "loss": 2.9552, + "theoretical_loss": 3.99309012862572, + "tokens_seen": 417267712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411835506519559, + "loss": 2.7448, + "theoretical_loss": 3.993020467884011, + "tokens_seen": 417333248 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044117352056168507, + "loss": 2.6875, + "theoretical_loss": 3.9929508211430553, + "tokens_seen": 417398784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044116349047141425, + "loss": 2.7963, + "theoretical_loss": 3.9928811883978392, + "tokens_seen": 417464320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411534603811435, + "loss": 2.988, + "theoretical_loss": 3.9928115696433553, + "tokens_seen": 417529856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411434302908726, + "loss": 2.705, + "theoretical_loss": 3.992741964874596, + "tokens_seen": 417595392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044113340020060184, + "loss": 2.7923, + "theoretical_loss": 3.9926723740865584, + "tokens_seen": 417660928 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044112337011033097, + "loss": 2.6621, + "theoretical_loss": 3.992602797274241, + "tokens_seen": 417726464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 241540, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0460522174835205, + "objective/train/theoretical_loss": 3.9925332344326447, + "objective/train/tokens_used": 438252000, + "theoretical_loss": 3.9925332344326447, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411133400200602, + "loss": 2.5784, + "theoretical_loss": 3.9925332344326447, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411033099297894, + "loss": 3.0353, + "theoretical_loss": 3.9924636855567743, + "tokens_seen": 417857536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044109327983951857, + "loss": 2.7198, + "theoretical_loss": 3.992394150641636, + "tokens_seen": 417923072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044108324974924775, + "loss": 2.956, + "theoretical_loss": 3.992324629682238, + "tokens_seen": 417988608 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044107321965897693, + "loss": 2.4868, + "theoretical_loss": 3.992255122673593, + "tokens_seen": 418054144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410631895687061, + "loss": 3.0218, + "theoretical_loss": 3.9921856296107143, + "tokens_seen": 418119680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044105315947843535, + "loss": 2.8273, + "theoretical_loss": 3.992116150488619, + "tokens_seen": 418185216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410431293881645, + "loss": 2.8603, + "theoretical_loss": 3.992046685302327, + "tokens_seen": 418250752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410330992978937, + "loss": 2.6586, + "theoretical_loss": 3.9919772340468596, + "tokens_seen": 418316288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044102306920762284, + "loss": 2.5768, + "theoretical_loss": 3.9919077967172414, + "tokens_seen": 418381824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044101303911735207, + "loss": 3.0208, + "theoretical_loss": 3.991838373308499, + "tokens_seen": 418447360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044100300902708125, + "loss": 3.1297, + "theoretical_loss": 3.9917689638156624, + "tokens_seen": 418512896 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044099297893681043, + "loss": 2.701, + "theoretical_loss": 3.9916995682337633, + "tokens_seen": 418578432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409829488465396, + "loss": 2.8845, + "theoretical_loss": 3.9916301865578365, + "tokens_seen": 418643968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044097291875626885, + "loss": 2.8086, + "theoretical_loss": 3.9915608187829195, + "tokens_seen": 418709504 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440962888665998, + "loss": 2.9099, + "theoretical_loss": 3.991491464904052, + "tokens_seen": 418775040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409528585757272, + "loss": 2.9136, + "theoretical_loss": 3.9914221249162756, + "tokens_seen": 418840576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094282848545634, + "loss": 2.8021, + "theoretical_loss": 3.9913527988146353, + "tokens_seen": 418906112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409327983951856, + "loss": 2.8487, + "theoretical_loss": 3.9912834865941793, + "tokens_seen": 418971648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044092276830491476, + "loss": 2.9562, + "theoretical_loss": 3.9912141882499563, + "tokens_seen": 419037184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044091273821464394, + "loss": 2.9475, + "theoretical_loss": 3.99114490377702, + "tokens_seen": 419102720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409027081243731, + "loss": 2.7868, + "theoretical_loss": 3.991075633170425, + "tokens_seen": 419168256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408926780341023, + "loss": 2.9248, + "theoretical_loss": 3.991006376425228, + "tokens_seen": 419233792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408826479438315, + "loss": 2.8554, + "theoretical_loss": 3.99093713353649, + "tokens_seen": 419299328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408726178535607, + "loss": 2.6956, + "theoretical_loss": 3.9908679044992725, + "tokens_seen": 419364864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 242971, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7546820640563965, + "objective/train/theoretical_loss": 3.990798689308642, + "objective/train/tokens_used": 439890400, + "theoretical_loss": 3.990798689308642, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086258776328984, + "loss": 2.9533, + "theoretical_loss": 3.990798689308642, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408525576730191, + "loss": 3.0452, + "theoretical_loss": 3.9907294879596655, + "tokens_seen": 419495936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408425275827482, + "loss": 2.8135, + "theoretical_loss": 3.9906603004474137, + "tokens_seen": 419561472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044083249749247744, + "loss": 2.7665, + "theoretical_loss": 3.990591126766958, + "tokens_seen": 419627008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408224674022066, + "loss": 2.6103, + "theoretical_loss": 3.9905219669133745, + "tokens_seen": 419692544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408124373119358, + "loss": 3.1082, + "theoretical_loss": 3.9904528208817407, + "tokens_seen": 419758080 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440802407221665, + "loss": 2.7595, + "theoretical_loss": 3.990383688667137, + "tokens_seen": 419823616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407923771313942, + "loss": 3.0086, + "theoretical_loss": 3.9903145702646468, + "tokens_seen": 419889152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044078234704112335, + "loss": 2.8238, + "theoretical_loss": 3.9902454656693545, + "tokens_seen": 419954688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407723169508526, + "loss": 2.9094, + "theoretical_loss": 3.990176374876348, + "tokens_seen": 420020224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407622868605817, + "loss": 3.11, + "theoretical_loss": 3.9901072978807184, + "tokens_seen": 420085760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044075225677031094, + "loss": 2.8017, + "theoretical_loss": 3.9900382346775576, + "tokens_seen": 420151296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407422266800401, + "loss": 3.0109, + "theoretical_loss": 3.9899691852619616, + "tokens_seen": 420216832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407321965897693, + "loss": 3.0303, + "theoretical_loss": 3.9899001496290274, + "tokens_seen": 420282368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407221664994985, + "loss": 2.7646, + "theoretical_loss": 3.989831127773856, + "tokens_seen": 420347904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044071213640922767, + "loss": 2.8605, + "theoretical_loss": 3.989762119691551, + "tokens_seen": 420413440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044070210631895685, + "loss": 2.8231, + "theoretical_loss": 3.9896931253772165, + "tokens_seen": 420478976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406920762286861, + "loss": 2.5455, + "theoretical_loss": 3.989624144825961, + "tokens_seen": 420544512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044068204613841527, + "loss": 3.1163, + "theoretical_loss": 3.9895551780328953, + "tokens_seen": 420610048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044067201604814445, + "loss": 2.6076, + "theoretical_loss": 3.989486224993131, + "tokens_seen": 420675584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406619859578737, + "loss": 3.1176, + "theoretical_loss": 3.9894172857017844, + "tokens_seen": 420741120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406519558676028, + "loss": 3.0003, + "theoretical_loss": 3.989348360153974, + "tokens_seen": 420806656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044064192577733204, + "loss": 2.955, + "theoretical_loss": 3.989279448344818, + "tokens_seen": 420872192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044063189568706117, + "loss": 2.8199, + "theoretical_loss": 3.9892105502694415, + "tokens_seen": 420937728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406218655967904, + "loss": 2.9728, + "theoretical_loss": 3.989141665922969, + "tokens_seen": 421003264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 244183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9032411575317383, + "objective/train/theoretical_loss": 3.989072795300528, + "objective/train/tokens_used": 441528800, + "theoretical_loss": 3.989072795300528, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406118355065196, + "loss": 2.8608, + "theoretical_loss": 3.989072795300528, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044060180541624877, + "loss": 2.9812, + "theoretical_loss": 3.989003938397249, + "tokens_seen": 421134336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044059177532597795, + "loss": 2.9128, + "theoretical_loss": 3.988935095208266, + "tokens_seen": 421199872 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044058174523570713, + "loss": 2.8998, + "theoretical_loss": 3.988866265728712, + "tokens_seen": 421265408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405717151454363, + "loss": 2.7796, + "theoretical_loss": 3.9887974499537266, + "tokens_seen": 421330944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044056168505516555, + "loss": 2.9465, + "theoretical_loss": 3.9887286478784496, + "tokens_seen": 421396480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405516549648947, + "loss": 2.6366, + "theoretical_loss": 3.988659859498023, + "tokens_seen": 421462016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405416248746239, + "loss": 2.8122, + "theoretical_loss": 3.9885910848075934, + "tokens_seen": 421527552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044053159478435304, + "loss": 2.9463, + "theoretical_loss": 3.9885223238023073, + "tokens_seen": 421593088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044052156469408227, + "loss": 3.0525, + "theoretical_loss": 3.988453576477315, + "tokens_seen": 421658624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044051153460381145, + "loss": 2.7746, + "theoretical_loss": 3.9883848428277697, + "tokens_seen": 421724160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044050150451354063, + "loss": 2.7614, + "theoretical_loss": 3.988316122848826, + "tokens_seen": 421789696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404914744232698, + "loss": 2.9223, + "theoretical_loss": 3.988247416535642, + "tokens_seen": 421855232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044048144433299905, + "loss": 2.9403, + "theoretical_loss": 3.9881787238833772, + "tokens_seen": 421920768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404714142427282, + "loss": 2.7421, + "theoretical_loss": 3.9881100448871942, + "tokens_seen": 421986304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404613841524574, + "loss": 2.939, + "theoretical_loss": 3.988041379542258, + "tokens_seen": 422051840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044045135406218654, + "loss": 2.831, + "theoretical_loss": 3.9879727278437365, + "tokens_seen": 422117376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404413239719158, + "loss": 2.8991, + "theoretical_loss": 3.987904089786799, + "tokens_seen": 422182912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044043129388164496, + "loss": 2.7653, + "theoretical_loss": 3.9878354653666177, + "tokens_seen": 422248448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044042126379137414, + "loss": 2.9397, + "theoretical_loss": 3.987766854578368, + "tokens_seen": 422313984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404112337011033, + "loss": 3.1411, + "theoretical_loss": 3.987698257417227, + "tokens_seen": 422379520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404012036108325, + "loss": 3.0797, + "theoretical_loss": 3.9876296738783745, + "tokens_seen": 422445056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403911735205617, + "loss": 2.7101, + "theoretical_loss": 3.987561103956992, + "tokens_seen": 422510592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403811434302909, + "loss": 2.8034, + "theoretical_loss": 3.987492547648265, + "tokens_seen": 422576128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044037111334002004, + "loss": 2.8983, + "theoretical_loss": 3.98742400494738, + "tokens_seen": 422641664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 244758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9229331016540527, + "objective/train/theoretical_loss": 3.987355475849527, + "objective/train/tokens_used": 443167200, + "theoretical_loss": 3.987355475849527, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403610832497493, + "loss": 3.0326, + "theoretical_loss": 3.987355475849527, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403510531594784, + "loss": 2.7311, + "theoretical_loss": 3.9872869603498975, + "tokens_seen": 422772736 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044034102306920764, + "loss": 2.9594, + "theoretical_loss": 3.987218458443686, + "tokens_seen": 422838272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403309929789368, + "loss": 2.9823, + "theoretical_loss": 3.9871499701260897, + "tokens_seen": 422903808 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440320962888666, + "loss": 2.9567, + "theoretical_loss": 3.9870814953923075, + "tokens_seen": 422969344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403109327983952, + "loss": 2.9457, + "theoretical_loss": 3.987013034237541, + "tokens_seen": 423034880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403009027081244, + "loss": 2.9932, + "theoretical_loss": 3.9869445866569944, + "tokens_seen": 423100416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044029087261785355, + "loss": 2.9196, + "theoretical_loss": 3.986876152645875, + "tokens_seen": 423165952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402808425275828, + "loss": 2.7942, + "theoretical_loss": 3.9868077321993916, + "tokens_seen": 423231488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402708124373119, + "loss": 2.8662, + "theoretical_loss": 3.986739325312755, + "tokens_seen": 423297024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044026078234704114, + "loss": 2.9557, + "theoretical_loss": 3.98667093198118, + "tokens_seen": 423362560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402507522567703, + "loss": 3.1629, + "theoretical_loss": 3.986602552199882, + "tokens_seen": 423428096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402407221664995, + "loss": 2.9203, + "theoretical_loss": 3.9865341859640804, + "tokens_seen": 423493632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402306920762287, + "loss": 3.0048, + "theoretical_loss": 3.986465833268996, + "tokens_seen": 423559168 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044022066198595787, + "loss": 2.7389, + "theoretical_loss": 3.986397494109853, + "tokens_seen": 423624704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044021063189568705, + "loss": 2.9356, + "theoretical_loss": 3.986329168481877, + "tokens_seen": 423690240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402006018054163, + "loss": 2.8892, + "theoretical_loss": 3.986260856380296, + "tokens_seen": 423755776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401905717151454, + "loss": 2.799, + "theoretical_loss": 3.9861925578003428, + "tokens_seen": 423821312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044018054162487465, + "loss": 2.9222, + "theoretical_loss": 3.986124272737248, + "tokens_seen": 423886848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044017051153460383, + "loss": 2.98, + "theoretical_loss": 3.986056001186249, + "tokens_seen": 423952384 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440160481444333, + "loss": 2.9878, + "theoretical_loss": 3.9859877431425836, + "tokens_seen": 424017920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401504513540622, + "loss": 2.935, + "theoretical_loss": 3.9859194986014925, + "tokens_seen": 424083456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044014042126379137, + "loss": 2.8919, + "theoretical_loss": 3.9858512675582185, + "tokens_seen": 424148992 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044013039117352055, + "loss": 2.856, + "theoretical_loss": 3.9857830500080067, + "tokens_seen": 424214528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401203610832498, + "loss": 2.9074, + "theoretical_loss": 3.985714845946105, + "tokens_seen": 424280064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 245792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5828306674957275, + "objective/train/theoretical_loss": 3.9856466553677645, + "objective/train/tokens_used": 444805600, + "theoretical_loss": 3.9856466553677645, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401103309929789, + "loss": 2.7627, + "theoretical_loss": 3.9856466553677645, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044010030090270815, + "loss": 2.9771, + "theoretical_loss": 3.985578478268236, + "tokens_seen": 424411136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400902708124373, + "loss": 2.7067, + "theoretical_loss": 3.985510314642776, + "tokens_seen": 424476672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400802407221665, + "loss": 3.1371, + "theoretical_loss": 3.985442164486641, + "tokens_seen": 424542208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400702106318957, + "loss": 2.7251, + "theoretical_loss": 3.9853740277950918, + "tokens_seen": 424607744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400601805416249, + "loss": 2.7574, + "theoretical_loss": 3.98530590456339, + "tokens_seen": 424673280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044005015045135406, + "loss": 2.8466, + "theoretical_loss": 3.985237794786799, + "tokens_seen": 424738816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044004012036108324, + "loss": 3.1828, + "theoretical_loss": 3.985169698460588, + "tokens_seen": 424804352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400300902708124, + "loss": 2.867, + "theoretical_loss": 3.9851016155800254, + "tokens_seen": 424869888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044002006018054165, + "loss": 2.7883, + "theoretical_loss": 3.9850335461403823, + "tokens_seen": 424935424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400100300902708, + "loss": 2.8638, + "theoretical_loss": 3.984965490136934, + "tokens_seen": 425000960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044, + "loss": 2.989, + "theoretical_loss": 3.9848974475649563, + "tokens_seen": 425066496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399899699097292, + "loss": 3.2045, + "theoretical_loss": 3.9848294184197286, + "tokens_seen": 425132032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399799398194584, + "loss": 2.8111, + "theoretical_loss": 3.984761402696532, + "tokens_seen": 425197568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043996990972918756, + "loss": 2.8473, + "theoretical_loss": 3.9846934003906505, + "tokens_seen": 425263104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043995987963891674, + "loss": 2.684, + "theoretical_loss": 3.98462541149737, + "tokens_seen": 425328640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399498495486459, + "loss": 2.6684, + "theoretical_loss": 3.984557436011978, + "tokens_seen": 425394176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043993981945837516, + "loss": 2.8611, + "theoretical_loss": 3.9844894739297674, + "tokens_seen": 425459712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043992978936810434, + "loss": 2.8828, + "theoretical_loss": 3.98442152524603, + "tokens_seen": 425525248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399197592778335, + "loss": 2.7737, + "theoretical_loss": 3.984353589956062, + "tokens_seen": 425590784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399097291875627, + "loss": 3.0198, + "theoretical_loss": 3.9842856680551604, + "tokens_seen": 425656320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398996990972919, + "loss": 2.966, + "theoretical_loss": 3.9842177595386277, + "tokens_seen": 425721856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398896690070211, + "loss": 2.9014, + "theoretical_loss": 3.9841498644017643, + "tokens_seen": 425787392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043987963891675024, + "loss": 2.9488, + "theoretical_loss": 3.984081982639877, + "tokens_seen": 425852928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398696088264795, + "loss": 2.9806, + "theoretical_loss": 3.9840141142482723, + "tokens_seen": 425918464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 246485, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.934349298477173, + "objective/train/theoretical_loss": 3.98394625922226, + "objective/train/tokens_used": 446444000, + "theoretical_loss": 3.98394625922226, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398595787362086, + "loss": 2.8034, + "theoretical_loss": 3.98394625922226, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043984954864593784, + "loss": 2.9262, + "theoretical_loss": 3.983878417557153, + "tokens_seen": 426049536 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439839518555667, + "loss": 2.9786, + "theoretical_loss": 3.9838105892482663, + "tokens_seen": 426115072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398294884653962, + "loss": 2.6863, + "theoretical_loss": 3.9837427742909153, + "tokens_seen": 426180608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398194583751254, + "loss": 2.749, + "theoretical_loss": 3.9836749726804204, + "tokens_seen": 426246144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398094282848546, + "loss": 2.8953, + "theoretical_loss": 3.983607184412103, + "tokens_seen": 426311680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043979939819458375, + "loss": 2.841, + "theoretical_loss": 3.9835394094812875, + "tokens_seen": 426377216 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439789368104313, + "loss": 2.7696, + "theoretical_loss": 3.9834716478832997, + "tokens_seen": 426442752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397793380140421, + "loss": 2.7188, + "theoretical_loss": 3.9834038996134686, + "tokens_seen": 426508288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043976930792377134, + "loss": 2.717, + "theoretical_loss": 3.9833361646671257, + "tokens_seen": 426573824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397592778335005, + "loss": 2.977, + "theoretical_loss": 3.9832684430396035, + "tokens_seen": 426639360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397492477432297, + "loss": 2.9569, + "theoretical_loss": 3.983200734726238, + "tokens_seen": 426704896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397392176529589, + "loss": 2.6681, + "theoretical_loss": 3.9831330397223685, + "tokens_seen": 426770432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043972918756268807, + "loss": 2.8142, + "theoretical_loss": 3.9830653580233344, + "tokens_seen": 426835968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043971915747241725, + "loss": 2.796, + "theoretical_loss": 3.9829976896244794, + "tokens_seen": 426901504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397091273821465, + "loss": 2.91, + "theoretical_loss": 3.9829300345211474, + "tokens_seen": 426967040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396990972918756, + "loss": 2.8034, + "theoretical_loss": 3.9828623927086872, + "tokens_seen": 427032576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043968906720160485, + "loss": 2.6024, + "theoretical_loss": 3.9827947641824473, + "tokens_seen": 427098112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043967903711133403, + "loss": 2.9885, + "theoretical_loss": 3.982727148937782, + "tokens_seen": 427163648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396690070210632, + "loss": 3.0306, + "theoretical_loss": 3.982659546970044, + "tokens_seen": 427229184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396589769307924, + "loss": 2.9307, + "theoretical_loss": 3.9825919582745906, + "tokens_seen": 427294720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043964894684052157, + "loss": 3.1782, + "theoretical_loss": 3.9825243828467816, + "tokens_seen": 427360256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043963891675025075, + "loss": 2.7458, + "theoretical_loss": 3.982456820681978, + "tokens_seen": 427425792 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043962888665998, + "loss": 2.9876, + "theoretical_loss": 3.982389271775544, + "tokens_seen": 427491328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396188565697091, + "loss": 2.8233, + "theoretical_loss": 3.9823217361228456, + "tokens_seen": 427556864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 247615, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7162137031555176, + "objective/train/theoretical_loss": 3.982254213719252, + "objective/train/tokens_used": 448082400, + "theoretical_loss": 3.982254213719252, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043960882647943835, + "loss": 2.8383, + "theoretical_loss": 3.982254213719252, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395987963891675, + "loss": 2.8669, + "theoretical_loss": 3.9821867045601325, + "tokens_seen": 427687936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395887662988967, + "loss": 2.893, + "theoretical_loss": 3.9821192086408623, + "tokens_seen": 427753472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395787362086259, + "loss": 2.8802, + "theoretical_loss": 3.982051725956816, + "tokens_seen": 427819008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395687061183551, + "loss": 2.587, + "theoretical_loss": 3.9819842565033703, + "tokens_seen": 427884544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043955867602808426, + "loss": 2.9512, + "theoretical_loss": 3.9819168002759078, + "tokens_seen": 427950080 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043954864593781344, + "loss": 2.7715, + "theoretical_loss": 3.9818493572698093, + "tokens_seen": 428015616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395386158475426, + "loss": 3.0963, + "theoretical_loss": 3.9817819274804602, + "tokens_seen": 428081152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043952858575727185, + "loss": 2.8617, + "theoretical_loss": 3.9817145109032475, + "tokens_seen": 428146688 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439518555667001, + "loss": 2.8287, + "theoretical_loss": 3.9816471075335604, + "tokens_seen": 428212224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395085255767302, + "loss": 2.9118, + "theoretical_loss": 3.981579717366791, + "tokens_seen": 428277760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394984954864594, + "loss": 3.141, + "theoretical_loss": 3.981512340398333, + "tokens_seen": 428343296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394884653961886, + "loss": 2.6428, + "theoretical_loss": 3.9814449766235835, + "tokens_seen": 428408832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043947843530591776, + "loss": 2.9725, + "theoretical_loss": 3.9813776260379408, + "tokens_seen": 428474368 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043946840521564694, + "loss": 2.7616, + "theoretical_loss": 3.9813102886368057, + "tokens_seen": 428539904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394583751253761, + "loss": 2.8903, + "theoretical_loss": 3.9812429644155816, + "tokens_seen": 428605440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043944834503510536, + "loss": 3.0138, + "theoretical_loss": 3.9811756533696743, + "tokens_seen": 428670976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394383149448345, + "loss": 2.93, + "theoretical_loss": 3.9811083554944915, + "tokens_seen": 428736512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394282848545637, + "loss": 2.7467, + "theoretical_loss": 3.9810410707854436, + "tokens_seen": 428802048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043941825476429285, + "loss": 3.1358, + "theoretical_loss": 3.980973799237943, + "tokens_seen": 428867584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394082246740221, + "loss": 2.8636, + "theoretical_loss": 3.9809065408474047, + "tokens_seen": 428933120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043939819458375126, + "loss": 3.0446, + "theoretical_loss": 3.980839295609246, + "tokens_seen": 428998656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043938816449348044, + "loss": 2.7096, + "theoretical_loss": 3.9807720635188857, + "tokens_seen": 429064192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393781344032096, + "loss": 2.8974, + "theoretical_loss": 3.9807048445717457, + "tokens_seen": 429129728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393681043129388, + "loss": 3.0659, + "theoretical_loss": 3.9806376387632505, + "tokens_seen": 429195264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 248896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7603349685668945, + "objective/train/theoretical_loss": 3.9805704460888256, + "objective/train/tokens_used": 449720800, + "theoretical_loss": 3.9805704460888256, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439358074222668, + "loss": 2.9696, + "theoretical_loss": 3.9805704460888256, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393480441323972, + "loss": 3.04, + "theoretical_loss": 3.9805032665439004, + "tokens_seen": 429326336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933801404212635, + "loss": 2.7513, + "theoretical_loss": 3.9804361001239057, + "tokens_seen": 429391872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393279839518556, + "loss": 2.8466, + "theoretical_loss": 3.9803689468242744, + "tokens_seen": 429457408 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043931795386158477, + "loss": 2.7847, + "theoretical_loss": 3.9803018066404414, + "tokens_seen": 429522944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043930792377131395, + "loss": 2.9597, + "theoretical_loss": 3.980234679567845, + "tokens_seen": 429588480 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043929789368104313, + "loss": 2.878, + "theoretical_loss": 3.980167565601925, + "tokens_seen": 429654016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392878635907723, + "loss": 3.0399, + "theoretical_loss": 3.9801004647381246, + "tokens_seen": 429719552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392778335005015, + "loss": 3.145, + "theoretical_loss": 3.980033376971887, + "tokens_seen": 429785088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392678034102307, + "loss": 3.0189, + "theoretical_loss": 3.97996630229866, + "tokens_seen": 429850624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043925777331995985, + "loss": 2.8291, + "theoretical_loss": 3.9798992407138924, + "tokens_seen": 429916160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392477432296891, + "loss": 2.8082, + "theoretical_loss": 3.9798321922130357, + "tokens_seen": 429981696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392377131394182, + "loss": 2.8025, + "theoretical_loss": 3.979765156791543, + "tokens_seen": 430047232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043922768304914745, + "loss": 2.8277, + "theoretical_loss": 3.979698134444871, + "tokens_seen": 430112768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921765295887663, + "loss": 2.9302, + "theoretical_loss": 3.979631125168478, + "tokens_seen": 430178304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392076228686058, + "loss": 2.7819, + "theoretical_loss": 3.979564128957824, + "tokens_seen": 430243840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439197592778335, + "loss": 2.887, + "theoretical_loss": 3.979497145808372, + "tokens_seen": 430309376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043918756268806423, + "loss": 2.8212, + "theoretical_loss": 3.979430175715587, + "tokens_seen": 430374912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391775325977934, + "loss": 3.0824, + "theoretical_loss": 3.979363218674936, + "tokens_seen": 430440448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391675025075226, + "loss": 2.8863, + "theoretical_loss": 3.9792962746818894, + "tokens_seen": 430505984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043915747241725177, + "loss": 2.9136, + "theoretical_loss": 3.9792293437319177, + "tokens_seen": 430571520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043914744232698095, + "loss": 2.8946, + "theoretical_loss": 3.979162425820496, + "tokens_seen": 430637056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391374122367102, + "loss": 2.8152, + "theoretical_loss": 3.9790955209431007, + "tokens_seen": 430702592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391273821464393, + "loss": 2.5009, + "theoretical_loss": 3.9790286290952097, + "tokens_seen": 430768128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043911735205616855, + "loss": 2.7643, + "theoretical_loss": 3.978961750272304, + "tokens_seen": 430833664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 249330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5802011489868164, + "objective/train/theoretical_loss": 3.978894884469867, + "objective/train/tokens_used": 451359200, + "theoretical_loss": 3.978894884469867, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391073219658977, + "loss": 2.6646, + "theoretical_loss": 3.978894884469867, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390972918756269, + "loss": 2.8741, + "theoretical_loss": 3.978828031683384, + "tokens_seen": 430964736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390872617853561, + "loss": 3.135, + "theoretical_loss": 3.978761191908343, + "tokens_seen": 431030272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390772316950853, + "loss": 2.7624, + "theoretical_loss": 3.9786943651402327, + "tokens_seen": 431095808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043906720160481446, + "loss": 2.8756, + "theoretical_loss": 3.9786275513745464, + "tokens_seen": 431161344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043905717151454364, + "loss": 2.8513, + "theoretical_loss": 3.9785607506067775, + "tokens_seen": 431226880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390471414242728, + "loss": 2.951, + "theoretical_loss": 3.978493962832424, + "tokens_seen": 431292416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043903711133400205, + "loss": 2.7171, + "theoretical_loss": 3.9784271880469824, + "tokens_seen": 431357952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390270812437312, + "loss": 2.7085, + "theoretical_loss": 3.978360426245956, + "tokens_seen": 431423488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390170511534604, + "loss": 2.8883, + "theoretical_loss": 3.9782936774248476, + "tokens_seen": 431489024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390070210631896, + "loss": 2.7489, + "theoretical_loss": 3.978226941579162, + "tokens_seen": 431554560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389969909729188, + "loss": 2.8769, + "theoretical_loss": 3.9781602187044074, + "tokens_seen": 431620096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043898696088264796, + "loss": 2.9323, + "theoretical_loss": 3.9780935087960945, + "tokens_seen": 431685632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043897693079237714, + "loss": 2.8336, + "theoretical_loss": 3.978026811849735, + "tokens_seen": 431751168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389669007021063, + "loss": 2.4792, + "theoretical_loss": 3.977960127860843, + "tokens_seen": 431816704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043895687061183556, + "loss": 2.7761, + "theoretical_loss": 3.977893456824936, + "tokens_seen": 431882240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389468405215647, + "loss": 2.847, + "theoretical_loss": 3.9778267987375324, + "tokens_seen": 431947776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389368104312939, + "loss": 2.9699, + "theoretical_loss": 3.9777601535941542, + "tokens_seen": 432013312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043892678034102305, + "loss": 2.9322, + "theoretical_loss": 3.977693521390324, + "tokens_seen": 432078848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389167502507523, + "loss": 2.7251, + "theoretical_loss": 3.9776269021215676, + "tokens_seen": 432144384 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043890672016048146, + "loss": 2.7903, + "theoretical_loss": 3.9775602957834133, + "tokens_seen": 432209920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043889669007021064, + "loss": 3.0632, + "theoretical_loss": 3.977493702371391, + "tokens_seen": 432275456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388866599799398, + "loss": 2.9212, + "theoretical_loss": 3.977427121881033, + "tokens_seen": 432340992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438876629889669, + "loss": 2.8574, + "theoretical_loss": 3.977360554307874, + "tokens_seen": 432406528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388665997993982, + "loss": 2.6996, + "theoretical_loss": 3.9772939996474506, + "tokens_seen": 432472064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 250327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1851444244384766, + "objective/train/theoretical_loss": 3.9772274578953013, + "objective/train/tokens_used": 452997600, + "theoretical_loss": 3.9772274578953013, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388565697091274, + "loss": 2.49, + "theoretical_loss": 3.9772274578953013, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043884653961885655, + "loss": 3.0437, + "theoretical_loss": 3.977160929046968, + "tokens_seen": 432603136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388365095285858, + "loss": 3.0128, + "theoretical_loss": 3.977094413097994, + "tokens_seen": 432668672 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043882647943831497, + "loss": 3.064, + "theoretical_loss": 3.9770279100439256, + "tokens_seen": 432734208 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043881644934804415, + "loss": 3.0597, + "theoretical_loss": 3.9769614198803094, + "tokens_seen": 432799744 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043880641925777333, + "loss": 2.7542, + "theoretical_loss": 3.976894942602696, + "tokens_seen": 432865280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387963891675025, + "loss": 2.9004, + "theoretical_loss": 3.976828478206638, + "tokens_seen": 432930816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387863590772317, + "loss": 2.7999, + "theoretical_loss": 3.976762026687689, + "tokens_seen": 432996352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387763289869609, + "loss": 2.9483, + "theoretical_loss": 3.9766955880414065, + "tokens_seen": 433061888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043876629889669005, + "loss": 2.7133, + "theoretical_loss": 3.9766291622633494, + "tokens_seen": 433127424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387562688064193, + "loss": 2.8788, + "theoretical_loss": 3.9765627493490783, + "tokens_seen": 433192960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387462387161484, + "loss": 2.7892, + "theoretical_loss": 3.976496349294157, + "tokens_seen": 433258496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043873620862587765, + "loss": 2.8578, + "theoretical_loss": 3.9764299620941506, + "tokens_seen": 433324032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043872617853560683, + "loss": 2.86, + "theoretical_loss": 3.9763635877446273, + "tokens_seen": 433389568 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438716148445336, + "loss": 2.8879, + "theoretical_loss": 3.9762972262411562, + "tokens_seen": 433455104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387061183550652, + "loss": 2.9078, + "theoretical_loss": 3.97623087757931, + "tokens_seen": 433520640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043869608826479443, + "loss": 2.9007, + "theoretical_loss": 3.9761645417546627, + "tokens_seen": 433586176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043868605817452356, + "loss": 2.7336, + "theoretical_loss": 3.9760982187627913, + "tokens_seen": 433651712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386760280842528, + "loss": 3.0301, + "theoretical_loss": 3.976031908599274, + "tokens_seen": 433717248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386659979939819, + "loss": 2.8723, + "theoretical_loss": 3.9759656112596913, + "tokens_seen": 433782784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043865596790371115, + "loss": 2.9686, + "theoretical_loss": 3.9758993267396265, + "tokens_seen": 433848320 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043864593781344033, + "loss": 2.7986, + "theoretical_loss": 3.9758330550346663, + "tokens_seen": 433913856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386359077231695, + "loss": 2.8748, + "theoretical_loss": 3.975766796140396, + "tokens_seen": 433979392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386258776328987, + "loss": 2.8568, + "theoretical_loss": 3.975700550052406, + "tokens_seen": 434044928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386158475426279, + "loss": 2.828, + "theoretical_loss": 3.9756343167662886, + "tokens_seen": 434110464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 251069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.99489426612854, + "objective/train/theoretical_loss": 3.9755680962776374, + "objective/train/tokens_used": 454636000, + "theoretical_loss": 3.9755680962776374, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043860581745235706, + "loss": 2.7617, + "theoretical_loss": 3.9755680962776374, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385957873620863, + "loss": 3.1528, + "theoretical_loss": 3.9755018885820483, + "tokens_seen": 434241536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385857572718154, + "loss": 2.9258, + "theoretical_loss": 3.97543569367512, + "tokens_seen": 434307072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043857572718154466, + "loss": 2.9792, + "theoretical_loss": 3.9753695115524534, + "tokens_seen": 434372608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385656970912738, + "loss": 2.7644, + "theoretical_loss": 3.97530334220965, + "tokens_seen": 434438144 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438555667001003, + "loss": 2.9931, + "theoretical_loss": 3.975237185642316, + "tokens_seen": 434503680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385456369107322, + "loss": 2.948, + "theoretical_loss": 3.9751710418460577, + "tokens_seen": 434569216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385356068204614, + "loss": 3.1, + "theoretical_loss": 3.9751049108164844, + "tokens_seen": 434634752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043852557673019056, + "loss": 2.8339, + "theoretical_loss": 3.9750387925492072, + "tokens_seen": 434700288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385155466399198, + "loss": 2.9765, + "theoretical_loss": 3.974972687039841, + "tokens_seen": 434765824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385055165496489, + "loss": 2.5896, + "theoretical_loss": 3.9749065942839996, + "tokens_seen": 434831360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043849548645937816, + "loss": 2.8618, + "theoretical_loss": 3.974840514277303, + "tokens_seen": 434896896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384854563691073, + "loss": 2.9577, + "theoretical_loss": 3.974774447015369, + "tokens_seen": 434962432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384754262788365, + "loss": 2.9189, + "theoretical_loss": 3.9747083924938216, + "tokens_seen": 435027968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384653961885657, + "loss": 2.779, + "theoretical_loss": 3.974642350708284, + "tokens_seen": 435093504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384553660982949, + "loss": 2.99, + "theoretical_loss": 3.974576321654384, + "tokens_seen": 435159040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043844533600802407, + "loss": 2.6761, + "theoretical_loss": 3.9745103053277493, + "tokens_seen": 435224576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043843530591775325, + "loss": 2.8512, + "theoretical_loss": 3.974444301724011, + "tokens_seen": 435290112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384252758274825, + "loss": 2.8829, + "theoretical_loss": 3.9743783108388024, + "tokens_seen": 435355648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043841524573721166, + "loss": 3.0582, + "theoretical_loss": 3.9743123326677585, + "tokens_seen": 435421184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043840521564694084, + "loss": 3.0055, + "theoretical_loss": 3.9742463672065167, + "tokens_seen": 435486720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043839518555667, + "loss": 2.7174, + "theoretical_loss": 3.9741804144507165, + "tokens_seen": 435552256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383851554663992, + "loss": 2.9559, + "theoretical_loss": 3.974114474395999, + "tokens_seen": 435617792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383751253761284, + "loss": 2.9737, + "theoretical_loss": 3.9740485470380094, + "tokens_seen": 435683328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383650952858576, + "loss": 2.8755, + "theoretical_loss": 3.973982632372392, + "tokens_seen": 435748864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 252128, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7361905574798584, + "objective/train/theoretical_loss": 3.973916730394796, + "objective/train/tokens_used": 456274400, + "theoretical_loss": 3.973916730394796, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043835506519558675, + "loss": 2.7758, + "theoretical_loss": 3.973916730394796, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438345035105316, + "loss": 2.9473, + "theoretical_loss": 3.9738508411008713, + "tokens_seen": 435879936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043833500501504517, + "loss": 2.8355, + "theoretical_loss": 3.97378496448627, + "tokens_seen": 435945472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043832497492477435, + "loss": 2.9717, + "theoretical_loss": 3.9737191005466475, + "tokens_seen": 436011008 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043831494483450353, + "loss": 2.8114, + "theoretical_loss": 3.973653249277659, + "tokens_seen": 436076544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383049147442327, + "loss": 2.9563, + "theoretical_loss": 3.973587410674965, + "tokens_seen": 436142080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382948846539619, + "loss": 3.0945, + "theoretical_loss": 3.973521584734225, + "tokens_seen": 436207616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382848545636911, + "loss": 3.0023, + "theoretical_loss": 3.9734557714511034, + "tokens_seen": 436273152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043827482447342025, + "loss": 3.158, + "theoretical_loss": 3.9733899708212643, + "tokens_seen": 436338688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382647943831495, + "loss": 2.7855, + "theoretical_loss": 3.973324182840375, + "tokens_seen": 436404224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382547642928786, + "loss": 2.7221, + "theoretical_loss": 3.9732584075041064, + "tokens_seen": 436469760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043824473420260785, + "loss": 2.9749, + "theoretical_loss": 3.9731926448081287, + "tokens_seen": 436535296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043823470411233703, + "loss": 2.8126, + "theoretical_loss": 3.973126894748116, + "tokens_seen": 436600832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382246740220662, + "loss": 2.909, + "theoretical_loss": 3.973061157319745, + "tokens_seen": 436666368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382146439317954, + "loss": 2.9275, + "theoretical_loss": 3.972995432518693, + "tokens_seen": 436731904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043820461384152463, + "loss": 2.8421, + "theoretical_loss": 3.9729297203406393, + "tokens_seen": 436797440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043819458375125376, + "loss": 2.7437, + "theoretical_loss": 3.9728640207812678, + "tokens_seen": 436862976 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438184553660983, + "loss": 2.7797, + "theoretical_loss": 3.9727983338362627, + "tokens_seen": 436928512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381745235707121, + "loss": 2.9771, + "theoretical_loss": 3.9727326595013093, + "tokens_seen": 436994048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043816449348044135, + "loss": 2.5017, + "theoretical_loss": 3.972666997772097, + "tokens_seen": 437059584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043815446339017053, + "loss": 3.0961, + "theoretical_loss": 3.972601348644317, + "tokens_seen": 437125120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381444332998997, + "loss": 2.7381, + "theoretical_loss": 3.9725357121136615, + "tokens_seen": 437190656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381344032096289, + "loss": 2.9324, + "theoretical_loss": 3.9724700881758253, + "tokens_seen": 437256192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381243731193581, + "loss": 2.7951, + "theoretical_loss": 3.9724044768265063, + "tokens_seen": 437321728 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043811434302908726, + "loss": 2.9698, + "theoretical_loss": 3.9723388780614033, + "tokens_seen": 437387264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 252698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6559126377105713, + "objective/train/theoretical_loss": 3.9722732918762174, + "objective/train/tokens_used": 457912800, + "theoretical_loss": 3.9722732918762174, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381043129388165, + "loss": 2.88, + "theoretical_loss": 3.9722732918762174, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380942828485456, + "loss": 2.7562, + "theoretical_loss": 3.9722077182666524, + "tokens_seen": 437518336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043808425275827486, + "loss": 2.9245, + "theoretical_loss": 3.9721421572284137, + "tokens_seen": 437583872 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438074222668004, + "loss": 2.7057, + "theoretical_loss": 3.972076608757209, + "tokens_seen": 437649408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380641925777332, + "loss": 3.2024, + "theoretical_loss": 3.9720110728487485, + "tokens_seen": 437714944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380541624874624, + "loss": 2.9399, + "theoretical_loss": 3.9719455494987432, + "tokens_seen": 437780480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380441323971916, + "loss": 2.8494, + "theoretical_loss": 3.971880038702908, + "tokens_seen": 437846016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043803410230692076, + "loss": 3.0082, + "theoretical_loss": 3.9718145404569585, + "tokens_seen": 437911552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043802407221665, + "loss": 2.8472, + "theoretical_loss": 3.971749054756613, + "tokens_seen": 437977088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380140421263791, + "loss": 2.8728, + "theoretical_loss": 3.971683581597592, + "tokens_seen": 438042624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043800401203610836, + "loss": 2.8619, + "theoretical_loss": 3.9716181209756174, + "tokens_seen": 438108160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379939819458375, + "loss": 2.8314, + "theoretical_loss": 3.9715526728864137, + "tokens_seen": 438173696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379839518555667, + "loss": 2.4957, + "theoretical_loss": 3.971487237325708, + "tokens_seen": 438239232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379739217652959, + "loss": 2.8016, + "theoretical_loss": 3.9714218142892292, + "tokens_seen": 438304768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379638916750251, + "loss": 2.8969, + "theoretical_loss": 3.9713564037727074, + "tokens_seen": 438370304 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043795386158475427, + "loss": 2.6609, + "theoretical_loss": 3.9712910057718758, + "tokens_seen": 438435840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043794383149448345, + "loss": 3.0509, + "theoretical_loss": 3.9712256202824694, + "tokens_seen": 438501376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043793380140421263, + "loss": 2.8158, + "theoretical_loss": 3.9711602473002245, + "tokens_seen": 438566912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043792377131394186, + "loss": 2.9429, + "theoretical_loss": 3.9710948868208815, + "tokens_seen": 438632448 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437913741223671, + "loss": 2.6593, + "theoretical_loss": 3.971029538840181, + "tokens_seen": 438697984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379037111334002, + "loss": 2.7673, + "theoretical_loss": 3.9709642033538666, + "tokens_seen": 438763520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043789368104312935, + "loss": 2.89, + "theoretical_loss": 3.970898880357683, + "tokens_seen": 438829056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378836509528586, + "loss": 2.8787, + "theoretical_loss": 3.9708335698473785, + "tokens_seen": 438894592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043787362086258777, + "loss": 2.8641, + "theoretical_loss": 3.9707682718187023, + "tokens_seen": 438960128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043786359077231695, + "loss": 2.7491, + "theoretical_loss": 3.9707029862674066, + "tokens_seen": 439025664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 253958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.985783815383911, + "objective/train/theoretical_loss": 3.970637713189244, + "objective/train/tokens_used": 459551200, + "theoretical_loss": 3.970637713189244, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785356068204613, + "loss": 2.9894, + "theoretical_loss": 3.970637713189244, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043784353059177537, + "loss": 2.6916, + "theoretical_loss": 3.9705724525799715, + "tokens_seen": 439156736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378335005015045, + "loss": 2.6918, + "theoretical_loss": 3.9705072044353455, + "tokens_seen": 439222272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043782347041123373, + "loss": 2.917, + "theoretical_loss": 3.970441968751128, + "tokens_seen": 439287808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781344032096286, + "loss": 2.8387, + "theoretical_loss": 3.9703767455230796, + "tokens_seen": 439353344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378034102306921, + "loss": 2.9011, + "theoretical_loss": 3.970311534746965, + "tokens_seen": 439418880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043779338014042127, + "loss": 2.774, + "theoretical_loss": 3.97024633641855, + "tokens_seen": 439484416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043778335005015045, + "loss": 2.9159, + "theoretical_loss": 3.970181150533603, + "tokens_seen": 439549952 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777331995987963, + "loss": 2.7145, + "theoretical_loss": 3.970115977087895, + "tokens_seen": 439615488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377632898696088, + "loss": 2.9115, + "theoretical_loss": 3.9700508160771966, + "tokens_seen": 439681024 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437753259779338, + "loss": 2.9576, + "theoretical_loss": 3.9699856674972844, + "tokens_seen": 439746560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043774322968906723, + "loss": 3.0514, + "theoretical_loss": 3.969920531343934, + "tokens_seen": 439812096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043773319959879636, + "loss": 2.7901, + "theoretical_loss": 3.9698554076129233, + "tokens_seen": 439877632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377231695085256, + "loss": 2.6878, + "theoretical_loss": 3.969790296300034, + "tokens_seen": 439943168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377131394182547, + "loss": 2.7006, + "theoretical_loss": 3.969725197401049, + "tokens_seen": 440008704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043770310932798396, + "loss": 2.9952, + "theoretical_loss": 3.969660110911752, + "tokens_seen": 440074240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043769307923771314, + "loss": 3.0273, + "theoretical_loss": 3.96959503682793, + "tokens_seen": 440139776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376830491474423, + "loss": 2.8135, + "theoretical_loss": 3.969529975145372, + "tokens_seen": 440205312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043767301905717155, + "loss": 2.7669, + "theoretical_loss": 3.9694649258598704, + "tokens_seen": 440270848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043766298896690073, + "loss": 2.767, + "theoretical_loss": 3.9693998889672164, + "tokens_seen": 440336384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376529588766299, + "loss": 2.7824, + "theoretical_loss": 3.969334864463206, + "tokens_seen": 440401920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376429287863591, + "loss": 2.6347, + "theoretical_loss": 3.9692698523436354, + "tokens_seen": 440467456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376328986960883, + "loss": 2.8143, + "theoretical_loss": 3.9692048526043044, + "tokens_seen": 440532992 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043762286860581746, + "loss": 3.0135, + "theoretical_loss": 3.969139865241014, + "tokens_seen": 440598528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376128385155467, + "loss": 2.7171, + "theoretical_loss": 3.9690748902495683, + "tokens_seen": 440664064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 254624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.297787666320801, + "objective/train/theoretical_loss": 3.9690099276257715, + "objective/train/tokens_used": 461189600, + "theoretical_loss": 3.9690099276257715, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376028084252758, + "loss": 2.9859, + "theoretical_loss": 3.9690099276257715, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043759277833500506, + "loss": 2.8268, + "theoretical_loss": 3.9689449773654317, + "tokens_seen": 440795136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375827482447342, + "loss": 2.5088, + "theoretical_loss": 3.9688800394643575, + "tokens_seen": 440860672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375727181544634, + "loss": 2.7787, + "theoretical_loss": 3.968815113918361, + "tokens_seen": 440926208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375626880641926, + "loss": 2.9048, + "theoretical_loss": 3.968750200723256, + "tokens_seen": 440991744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375526579739218, + "loss": 2.8828, + "theoretical_loss": 3.9686852998748567, + "tokens_seen": 441057280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043754262788365096, + "loss": 2.6297, + "theoretical_loss": 3.968620411368982, + "tokens_seen": 441122816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375325977933802, + "loss": 3.0614, + "theoretical_loss": 3.9685555352014505, + "tokens_seen": 441188352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375225677031093, + "loss": 2.958, + "theoretical_loss": 3.9684906713680848, + "tokens_seen": 441253888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043751253761283856, + "loss": 2.84, + "theoretical_loss": 3.9684258198647075, + "tokens_seen": 441319424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375025075225677, + "loss": 2.816, + "theoretical_loss": 3.9683609806871454, + "tokens_seen": 441384960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374924774322969, + "loss": 2.809, + "theoretical_loss": 3.9682961538312256, + "tokens_seen": 441450496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374824473420261, + "loss": 2.9887, + "theoretical_loss": 3.968231339292778, + "tokens_seen": 441516032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374724172517553, + "loss": 2.8036, + "theoretical_loss": 3.968166537067634, + "tokens_seen": 441581568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043746238716148447, + "loss": 2.7846, + "theoretical_loss": 3.9681017471516284, + "tokens_seen": 441647104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043745235707121365, + "loss": 2.7036, + "theoretical_loss": 3.968036969540596, + "tokens_seen": 441712640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043744232698094283, + "loss": 2.7011, + "theoretical_loss": 3.9679722042303753, + "tokens_seen": 441778176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043743229689067206, + "loss": 2.802, + "theoretical_loss": 3.967907451216806, + "tokens_seen": 441843712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374222668004012, + "loss": 2.9564, + "theoretical_loss": 3.9678427104957303, + "tokens_seen": 441909248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374122367101304, + "loss": 2.9014, + "theoretical_loss": 3.9677779820629917, + "tokens_seen": 441974784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043740220661985955, + "loss": 2.8714, + "theoretical_loss": 3.9677132659144365, + "tokens_seen": 442040320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373921765295888, + "loss": 2.8678, + "theoretical_loss": 3.9676485620459125, + "tokens_seen": 442105856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043738214643931797, + "loss": 2.789, + "theoretical_loss": 3.96758387045327, + "tokens_seen": 442171392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043737211634904715, + "loss": 2.6459, + "theoretical_loss": 3.967519191132361, + "tokens_seen": 442236928 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043736208625877633, + "loss": 3.0265, + "theoretical_loss": 3.9674545240790393, + "tokens_seen": 442302464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 255794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5010974407196045, + "objective/train/theoretical_loss": 3.967389869289161, + "objective/train/tokens_used": 462828000, + "theoretical_loss": 3.967389869289161, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043735205616850557, + "loss": 2.7137, + "theoretical_loss": 3.967389869289161, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373420260782347, + "loss": 2.6839, + "theoretical_loss": 3.967325226758584, + "tokens_seen": 442433536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043733199598796393, + "loss": 2.985, + "theoretical_loss": 3.967260596483169, + "tokens_seen": 442499072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043732196589769306, + "loss": 2.8362, + "theoretical_loss": 3.9671959784587774, + "tokens_seen": 442564608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373119358074223, + "loss": 2.7015, + "theoretical_loss": 3.9671313726812736, + "tokens_seen": 442630144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043730190571715147, + "loss": 2.957, + "theoretical_loss": 3.967066779146524, + "tokens_seen": 442695680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043729187562688065, + "loss": 2.9622, + "theoretical_loss": 3.9670021978503964, + "tokens_seen": 442761216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043728184553660983, + "loss": 2.7419, + "theoretical_loss": 3.966937628788761, + "tokens_seen": 442826752 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437271815446339, + "loss": 2.9077, + "theoretical_loss": 3.96687307195749, + "tokens_seen": 442892288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372617853560682, + "loss": 2.4599, + "theoretical_loss": 3.966808527352457, + "tokens_seen": 442957824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043725175526579743, + "loss": 2.9802, + "theoretical_loss": 3.966743994969539, + "tokens_seen": 443023360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043724172517552656, + "loss": 2.8803, + "theoretical_loss": 3.9666794748046135, + "tokens_seen": 443088896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372316950852558, + "loss": 2.752, + "theoretical_loss": 3.9666149668535615, + "tokens_seen": 443154432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372216649949849, + "loss": 2.9157, + "theoretical_loss": 3.9665504711122637, + "tokens_seen": 443219968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043721163490471416, + "loss": 2.8042, + "theoretical_loss": 3.9664859875766054, + "tokens_seen": 443285504 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043720160481444334, + "loss": 2.7464, + "theoretical_loss": 3.9664215162424723, + "tokens_seen": 443351040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371915747241725, + "loss": 2.9279, + "theoretical_loss": 3.9663570571057525, + "tokens_seen": 443416576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371815446339017, + "loss": 2.8611, + "theoretical_loss": 3.9662926101623364, + "tokens_seen": 443482112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043717151454363093, + "loss": 3.013, + "theoretical_loss": 3.9662281754081157, + "tokens_seen": 443547648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043716148445336006, + "loss": 2.9245, + "theoretical_loss": 3.9661637528389853, + "tokens_seen": 443613184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371514543630893, + "loss": 2.8922, + "theoretical_loss": 3.96609934245084, + "tokens_seen": 443678720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371414242728184, + "loss": 2.6798, + "theoretical_loss": 3.966034944239579, + "tokens_seen": 443744256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043713139418254766, + "loss": 2.6617, + "theoretical_loss": 3.965970558201102, + "tokens_seen": 443809792 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043712136409227684, + "loss": 2.8185, + "theoretical_loss": 3.9659061843313106, + "tokens_seen": 443875328 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437111334002006, + "loss": 2.7108, + "theoretical_loss": 3.9658418226261096, + "tokens_seen": 443940864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 257001, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6992599964141846, + "objective/train/theoretical_loss": 3.9657774730814044, + "objective/train/tokens_used": 464466400, + "theoretical_loss": 3.9657774730814044, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371013039117352, + "loss": 2.7786, + "theoretical_loss": 3.9657774730814044, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370912738214644, + "loss": 3.0641, + "theoretical_loss": 3.965713135693103, + "tokens_seen": 444071936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043708124373119356, + "loss": 2.8226, + "theoretical_loss": 3.9656488104571155, + "tokens_seen": 444137472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370712136409228, + "loss": 2.8475, + "theoretical_loss": 3.965584497369355, + "tokens_seen": 444203008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370611835506519, + "loss": 2.7757, + "theoretical_loss": 3.9655201964257336, + "tokens_seen": 444268544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043705115346038116, + "loss": 2.6841, + "theoretical_loss": 3.965455907622168, + "tokens_seen": 444334080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370411233701103, + "loss": 2.8601, + "theoretical_loss": 3.9653916309545765, + "tokens_seen": 444399616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370310932798395, + "loss": 2.8329, + "theoretical_loss": 3.9653273664188786, + "tokens_seen": 444465152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370210631895687, + "loss": 2.9373, + "theoretical_loss": 3.9652631140109955, + "tokens_seen": 444530688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370110330992979, + "loss": 2.8243, + "theoretical_loss": 3.965198873726852, + "tokens_seen": 444596224 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043700100300902707, + "loss": 3.0494, + "theoretical_loss": 3.965134645562374, + "tokens_seen": 444661760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369909729187563, + "loss": 3.087, + "theoretical_loss": 3.9650704295134878, + "tokens_seen": 444727296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043698094282848543, + "loss": 2.9131, + "theoretical_loss": 3.9650062255761247, + "tokens_seen": 444792832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043697091273821467, + "loss": 2.9083, + "theoretical_loss": 3.964942033746216, + "tokens_seen": 444858368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369608826479438, + "loss": 2.8564, + "theoretical_loss": 3.9648778540196945, + "tokens_seen": 444923904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043695085255767303, + "loss": 2.7252, + "theoretical_loss": 3.964813686392496, + "tokens_seen": 444989440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369408224674022, + "loss": 2.992, + "theoretical_loss": 3.9647495308605594, + "tokens_seen": 445054976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369307923771314, + "loss": 2.9756, + "theoretical_loss": 3.964685387419823, + "tokens_seen": 445120512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369207622868606, + "loss": 2.8825, + "theoretical_loss": 3.964621256066229, + "tokens_seen": 445186048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043691073219658975, + "loss": 2.804, + "theoretical_loss": 3.96455713679572, + "tokens_seen": 445251584 + }, + { + "epoch": 0.13, + "learning_rate": 0.000436900702106319, + "loss": 2.609, + "theoretical_loss": 3.9644930296042418, + "tokens_seen": 445317120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043689067201604817, + "loss": 2.6154, + "theoretical_loss": 3.964428934487742, + "tokens_seen": 445382656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043688064192577735, + "loss": 2.8911, + "theoretical_loss": 3.96436485144217, + "tokens_seen": 445448192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043687061183550653, + "loss": 2.9299, + "theoretical_loss": 3.964300780463477, + "tokens_seen": 445513728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043686058174523577, + "loss": 2.6714, + "theoretical_loss": 3.964236721547616, + "tokens_seen": 445579264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 257279, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.354438304901123, + "objective/train/theoretical_loss": 3.964172674690542, + "objective/train/tokens_used": 466104800, + "theoretical_loss": 3.964172674690542, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368505516549649, + "loss": 2.6825, + "theoretical_loss": 3.964172674690542, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043684052156469413, + "loss": 2.8716, + "theoretical_loss": 3.9641086398882126, + "tokens_seen": 445710336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043683049147442326, + "loss": 2.9858, + "theoretical_loss": 3.9640446171365866, + "tokens_seen": 445775872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368204613841525, + "loss": 2.7256, + "theoretical_loss": 3.9639806064316256, + "tokens_seen": 445841408 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043681043129388167, + "loss": 2.6819, + "theoretical_loss": 3.963916607769292, + "tokens_seen": 445906944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043680040120361085, + "loss": 3.0643, + "theoretical_loss": 3.963852621145551, + "tokens_seen": 445972480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043679037111334003, + "loss": 2.9543, + "theoretical_loss": 3.9637886465563685, + "tokens_seen": 446038016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367803410230692, + "loss": 3.0419, + "theoretical_loss": 3.9637246839977154, + "tokens_seen": 446103552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367703109327984, + "loss": 3.1, + "theoretical_loss": 3.9636607334655602, + "tokens_seen": 446169088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043676028084252763, + "loss": 2.8148, + "theoretical_loss": 3.9635967949558775, + "tokens_seen": 446234624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043675025075225676, + "loss": 2.8659, + "theoretical_loss": 3.963532868464641, + "tokens_seen": 446300160 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436740220661986, + "loss": 2.8417, + "theoretical_loss": 3.963468953987827, + "tokens_seen": 446365696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367301905717151, + "loss": 2.9253, + "theoretical_loss": 3.9634050515214145, + "tokens_seen": 446431232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043672016048144436, + "loss": 3.0207, + "theoretical_loss": 3.963341161061384, + "tokens_seen": 446496768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043671013039117354, + "loss": 2.7421, + "theoretical_loss": 3.9632772826037175, + "tokens_seen": 446562304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367001003009027, + "loss": 2.7032, + "theoretical_loss": 3.9632134161444, + "tokens_seen": 446627840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366900702106319, + "loss": 2.8559, + "theoretical_loss": 3.9631495616794172, + "tokens_seen": 446693376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043668004012036113, + "loss": 2.7473, + "theoretical_loss": 3.9630857192047566, + "tokens_seen": 446758912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043667001003009026, + "loss": 2.8741, + "theoretical_loss": 3.96302188871641, + "tokens_seen": 446824448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366599799398195, + "loss": 2.9051, + "theoretical_loss": 3.9629580702103686, + "tokens_seen": 446889984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366499498495486, + "loss": 2.9917, + "theoretical_loss": 3.9628942636826263, + "tokens_seen": 446955520 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043663991975927786, + "loss": 2.7779, + "theoretical_loss": 3.9628304691291785, + "tokens_seen": 447021056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043662988966900704, + "loss": 3.0163, + "theoretical_loss": 3.962766686546024, + "tokens_seen": 447086592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366198595787362, + "loss": 2.7074, + "theoretical_loss": 3.962702915929162, + "tokens_seen": 447152128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366098294884654, + "loss": 2.8268, + "theoretical_loss": 3.9626391572745945, + "tokens_seen": 447217664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 258449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.09433913230896, + "objective/train/theoretical_loss": 3.9625754105783244, + "objective/train/tokens_used": 467743200, + "theoretical_loss": 3.9625754105783244, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365997993981946, + "loss": 2.9999, + "theoretical_loss": 3.9625754105783244, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043658976930792376, + "loss": 2.6759, + "theoretical_loss": 3.9625116758363577, + "tokens_seen": 447348736 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436579739217653, + "loss": 3.012, + "theoretical_loss": 3.9624479530447023, + "tokens_seen": 447414272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656970912738213, + "loss": 2.8363, + "theoretical_loss": 3.962384242199367, + "tokens_seen": 447479808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043655967903711136, + "loss": 2.981, + "theoretical_loss": 3.962320543296363, + "tokens_seen": 447545344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365496489468405, + "loss": 2.9546, + "theoretical_loss": 3.9622568563317038, + "tokens_seen": 447610880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365396188565697, + "loss": 2.7905, + "theoretical_loss": 3.962193181301404, + "tokens_seen": 447676416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365295887662989, + "loss": 2.9201, + "theoretical_loss": 3.9621295182014813, + "tokens_seen": 447741952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365195586760281, + "loss": 2.8073, + "theoretical_loss": 3.9620658670279543, + "tokens_seen": 447807488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043650952858575727, + "loss": 3.0315, + "theoretical_loss": 3.9620022277768436, + "tokens_seen": 447873024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364994984954865, + "loss": 2.869, + "theoretical_loss": 3.9619386004441726, + "tokens_seen": 447938560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648946840521563, + "loss": 2.9296, + "theoretical_loss": 3.9618749850259647, + "tokens_seen": 448004096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043647943831494487, + "loss": 3.0547, + "theoretical_loss": 3.961811381518248, + "tokens_seen": 448069632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436469408224674, + "loss": 3.042, + "theoretical_loss": 3.96174778991705, + "tokens_seen": 448135168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043645937813440323, + "loss": 2.8965, + "theoretical_loss": 3.961684210218401, + "tokens_seen": 448200704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364493480441324, + "loss": 2.8506, + "theoretical_loss": 3.9616206424183336, + "tokens_seen": 448266240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364393179538616, + "loss": 2.8845, + "theoretical_loss": 3.961557086512882, + "tokens_seen": 448331776 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043642928786359077, + "loss": 2.824, + "theoretical_loss": 3.961493542498083, + "tokens_seen": 448397312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043641925777331995, + "loss": 2.9255, + "theoretical_loss": 3.9614300103699733, + "tokens_seen": 448462848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043640922768304913, + "loss": 2.9331, + "theoretical_loss": 3.961366490124593, + "tokens_seen": 448528384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043639919759277837, + "loss": 3.0854, + "theoretical_loss": 3.961302981757984, + "tokens_seen": 448593920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363891675025075, + "loss": 2.6079, + "theoretical_loss": 3.9612394852661907, + "tokens_seen": 448659456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043637913741223673, + "loss": 2.8454, + "theoretical_loss": 3.961176000645258, + "tokens_seen": 448724992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363691073219659, + "loss": 3.0051, + "theoretical_loss": 3.9611125278912334, + "tokens_seen": 448790528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363590772316951, + "loss": 2.703, + "theoretical_loss": 3.9610490670001663, + "tokens_seen": 448856064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 259697, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1037912368774414, + "objective/train/theoretical_loss": 3.9609856179681078, + "objective/train/tokens_used": 469381600, + "theoretical_loss": 3.9609856179681078, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363490471414243, + "loss": 3.0174, + "theoretical_loss": 3.9609856179681078, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043633901705115346, + "loss": 2.9102, + "theoretical_loss": 3.9609221807911115, + "tokens_seen": 448987136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632898696088264, + "loss": 2.832, + "theoretical_loss": 3.960858755465232, + "tokens_seen": 449052672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043631895687061187, + "loss": 3.0512, + "theoretical_loss": 3.9607953419865263, + "tokens_seen": 449118208 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436308926780341, + "loss": 2.8357, + "theoretical_loss": 3.960731940351053, + "tokens_seen": 449183744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043629889669007023, + "loss": 2.8914, + "theoretical_loss": 3.9606685505548738, + "tokens_seen": 449249280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043628886659979936, + "loss": 2.6328, + "theoretical_loss": 3.96060517259405, + "tokens_seen": 449314816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362788365095286, + "loss": 2.7281, + "theoretical_loss": 3.9605418064646467, + "tokens_seen": 449380352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362688064192578, + "loss": 3.0245, + "theoretical_loss": 3.9604784521627296, + "tokens_seen": 449445888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043625877632898696, + "loss": 3.0669, + "theoretical_loss": 3.9604151096843676, + "tokens_seen": 449511424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043624874623871614, + "loss": 2.9458, + "theoretical_loss": 3.9603517790256313, + "tokens_seen": 449576960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362387161484453, + "loss": 2.9036, + "theoretical_loss": 3.960288460182591, + "tokens_seen": 449642496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362286860581745, + "loss": 3.0075, + "theoretical_loss": 3.960225153151322, + "tokens_seen": 449708032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043621865596790374, + "loss": 2.8153, + "theoretical_loss": 3.9601618579279, + "tokens_seen": 449773568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043620862587763286, + "loss": 2.9289, + "theoretical_loss": 3.960098574508402, + "tokens_seen": 449839104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361985957873621, + "loss": 3.0163, + "theoretical_loss": 3.9600353028889073, + "tokens_seen": 449904640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361885656970913, + "loss": 3.0982, + "theoretical_loss": 3.9599720430654983, + "tokens_seen": 449970176 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043617853560682046, + "loss": 2.7447, + "theoretical_loss": 3.9599087950342566, + "tokens_seen": 450035712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361685055165497, + "loss": 2.9742, + "theoretical_loss": 3.959845558791269, + "tokens_seen": 450101248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361584754262788, + "loss": 3.0095, + "theoretical_loss": 3.9597823343326213, + "tokens_seen": 450166784 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043614844533600806, + "loss": 2.6936, + "theoretical_loss": 3.9597191216544028, + "tokens_seen": 450232320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043613841524573724, + "loss": 2.9194, + "theoretical_loss": 3.959655920752704, + "tokens_seen": 450297856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361283851554664, + "loss": 2.8928, + "theoretical_loss": 3.959592731623618, + "tokens_seen": 450363392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361183550651956, + "loss": 2.9382, + "theoretical_loss": 3.9595295542632387, + "tokens_seen": 450428928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361083249749248, + "loss": 2.8739, + "theoretical_loss": 3.9594663886676624, + "tokens_seen": 450494464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 260246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8196277618408203, + "objective/train/theoretical_loss": 3.9594032348329877, + "objective/train/tokens_used": 471020000, + "theoretical_loss": 3.9594032348329877, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043609829488465397, + "loss": 2.8036, + "theoretical_loss": 3.9594032348329877, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360882647943832, + "loss": 2.8566, + "theoretical_loss": 3.959340092755314, + "tokens_seen": 450625536 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043607823470411233, + "loss": 2.8465, + "theoretical_loss": 3.959276962430744, + "tokens_seen": 450691072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043606820461384156, + "loss": 2.9252, + "theoretical_loss": 3.9592138438553808, + "tokens_seen": 450756608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360581745235707, + "loss": 2.6681, + "theoretical_loss": 3.95915073702533, + "tokens_seen": 450822144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360481444332999, + "loss": 2.8351, + "theoretical_loss": 3.959087641936699, + "tokens_seen": 450887680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360381143430291, + "loss": 2.6263, + "theoretical_loss": 3.9590245585855977, + "tokens_seen": 450953216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360280842527583, + "loss": 3.0796, + "theoretical_loss": 3.9589614869681364, + "tokens_seen": 451018752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043601805416248747, + "loss": 2.8513, + "theoretical_loss": 3.9588984270804293, + "tokens_seen": 451084288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360080240722167, + "loss": 2.8436, + "theoretical_loss": 3.9588353789185895, + "tokens_seen": 451149824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043599799398194583, + "loss": 2.9357, + "theoretical_loss": 3.9587723424787358, + "tokens_seen": 451215360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043598796389167507, + "loss": 2.5351, + "theoretical_loss": 3.9587093177569854, + "tokens_seen": 451280896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359779338014042, + "loss": 3.0153, + "theoretical_loss": 3.958646304749459, + "tokens_seen": 451346432 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043596790371113343, + "loss": 2.7466, + "theoretical_loss": 3.9585833034522793, + "tokens_seen": 451411968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359578736208626, + "loss": 2.8312, + "theoretical_loss": 3.9585203138615697, + "tokens_seen": 451477504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359478435305918, + "loss": 2.7963, + "theoretical_loss": 3.9584573359734563, + "tokens_seen": 451543040 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043593781344032097, + "loss": 2.7758, + "theoretical_loss": 3.958394369784067, + "tokens_seen": 451608576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043592778335005015, + "loss": 2.8486, + "theoretical_loss": 3.9583314152895324, + "tokens_seen": 451674112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043591775325977933, + "loss": 2.956, + "theoretical_loss": 3.9582684724859822, + "tokens_seen": 451739648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043590772316950857, + "loss": 2.7166, + "theoretical_loss": 3.9582055413695514, + "tokens_seen": 451805184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358976930792377, + "loss": 3.0765, + "theoretical_loss": 3.9581426219363745, + "tokens_seen": 451870720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043588766298896693, + "loss": 2.679, + "theoretical_loss": 3.958079714182588, + "tokens_seen": 451936256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358776328986961, + "loss": 2.9488, + "theoretical_loss": 3.958016818104331, + "tokens_seen": 452001792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358676028084253, + "loss": 2.8143, + "theoretical_loss": 3.957953933697745, + "tokens_seen": 452067328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358575727181545, + "loss": 3.0067, + "theoretical_loss": 3.9578910609589713, + "tokens_seen": 452132864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 261277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.868929147720337, + "objective/train/theoretical_loss": 3.957828199884155, + "objective/train/tokens_used": 472658400, + "theoretical_loss": 3.957828199884155, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043584754262788366, + "loss": 2.8863, + "theoretical_loss": 3.957828199884155, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043583751253761284, + "loss": 2.8311, + "theoretical_loss": 3.9577653504694426, + "tokens_seen": 452263936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043582748244734207, + "loss": 2.8063, + "theoretical_loss": 3.9577025127109815, + "tokens_seen": 452329472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358174523570712, + "loss": 2.7761, + "theoretical_loss": 3.957639686604921, + "tokens_seen": 452395008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043580742226680043, + "loss": 2.8938, + "theoretical_loss": 3.9575768721474143, + "tokens_seen": 452460544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043579739217652956, + "loss": 3.0099, + "theoretical_loss": 3.9575140693346134, + "tokens_seen": 452526080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357873620862588, + "loss": 2.7552, + "theoretical_loss": 3.957451278162674, + "tokens_seen": 452591616 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435777331995988, + "loss": 2.6856, + "theoretical_loss": 3.957388498627754, + "tokens_seen": 452657152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043576730190571716, + "loss": 3.014, + "theoretical_loss": 3.957325730726012, + "tokens_seen": 452722688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043575727181544634, + "loss": 2.7964, + "theoretical_loss": 3.9572629744536085, + "tokens_seen": 452788224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357472417251755, + "loss": 2.9826, + "theoretical_loss": 3.9572002298067064, + "tokens_seen": 452853760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357372116349047, + "loss": 2.7544, + "theoretical_loss": 3.95713749678147, + "tokens_seen": 452919296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043572718154463394, + "loss": 2.8092, + "theoretical_loss": 3.957074775374066, + "tokens_seen": 452984832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043571715145436306, + "loss": 2.7585, + "theoretical_loss": 3.9570120655806615, + "tokens_seen": 453050368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357071213640923, + "loss": 2.8345, + "theoretical_loss": 3.956949367397427, + "tokens_seen": 453115904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356970912738215, + "loss": 2.9775, + "theoretical_loss": 3.956886680820535, + "tokens_seen": 453181440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043568706118355066, + "loss": 2.9825, + "theoretical_loss": 3.9568240058461575, + "tokens_seen": 453246976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043567703109327984, + "loss": 2.964, + "theoretical_loss": 3.956761342470471, + "tokens_seen": 453312512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435667001003009, + "loss": 2.8124, + "theoretical_loss": 3.9566986906896515, + "tokens_seen": 453378048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356569709127382, + "loss": 3.0178, + "theoretical_loss": 3.95663605049988, + "tokens_seen": 453443584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043564694082246744, + "loss": 2.9025, + "theoretical_loss": 3.9565734218973354, + "tokens_seen": 453509120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043563691073219657, + "loss": 2.8165, + "theoretical_loss": 3.9565108048782007, + "tokens_seen": 453574656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356268806419258, + "loss": 3.0191, + "theoretical_loss": 3.9564481994386607, + "tokens_seen": 453640192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043561685055165493, + "loss": 2.9861, + "theoretical_loss": 3.9563856055749014, + "tokens_seen": 453705728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043560682046138417, + "loss": 2.9445, + "theoretical_loss": 3.9563230232831113, + "tokens_seen": 453771264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 261864, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.966676712036133, + "objective/train/theoretical_loss": 3.956260452559479, + "objective/train/tokens_used": 474296800, + "theoretical_loss": 3.956260452559479, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043559679037111335, + "loss": 2.7952, + "theoretical_loss": 3.956260452559479, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043558676028084253, + "loss": 3.0193, + "theoretical_loss": 3.956197893400198, + "tokens_seen": 453902336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355767301905717, + "loss": 3.02, + "theoretical_loss": 3.95613534580146, + "tokens_seen": 453967872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355667001003009, + "loss": 2.995, + "theoretical_loss": 3.956072809759461, + "tokens_seen": 454033408 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043555667001003007, + "loss": 2.8422, + "theoretical_loss": 3.9560102852703976, + "tokens_seen": 454098944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355466399197593, + "loss": 2.8431, + "theoretical_loss": 3.955947772330469, + "tokens_seen": 454164480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043553660982948843, + "loss": 2.8094, + "theoretical_loss": 3.9558852709358763, + "tokens_seen": 454230016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043552657973921767, + "loss": 2.8377, + "theoretical_loss": 3.9558227810828215, + "tokens_seen": 454295552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043551654964894685, + "loss": 2.6956, + "theoretical_loss": 3.9557603027675086, + "tokens_seen": 454361088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043550651955867603, + "loss": 2.7347, + "theoretical_loss": 3.955697835986143, + "tokens_seen": 454426624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354964894684052, + "loss": 2.8711, + "theoretical_loss": 3.9556353807349343, + "tokens_seen": 454492160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354864593781344, + "loss": 2.9731, + "theoretical_loss": 3.955572937010091, + "tokens_seen": 454557696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354764292878636, + "loss": 2.93, + "theoretical_loss": 3.9555105048078243, + "tokens_seen": 454623232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354663991975928, + "loss": 2.7183, + "theoretical_loss": 3.9554480841243476, + "tokens_seen": 454688768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043545636910732194, + "loss": 2.9553, + "theoretical_loss": 3.9553856749558762, + "tokens_seen": 454754304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043544633901705117, + "loss": 2.8743, + "theoretical_loss": 3.9553232772986266, + "tokens_seen": 454819840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354363089267803, + "loss": 2.999, + "theoretical_loss": 3.955260891148818, + "tokens_seen": 454885376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043542627883650953, + "loss": 2.8236, + "theoretical_loss": 3.9551985165026693, + "tokens_seen": 454950912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043541624874623877, + "loss": 3.0874, + "theoretical_loss": 3.955136153356404, + "tokens_seen": 455016448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354062186559679, + "loss": 2.794, + "theoretical_loss": 3.955073801706246, + "tokens_seen": 455081984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043539618856569713, + "loss": 2.8035, + "theoretical_loss": 3.9550114615484198, + "tokens_seen": 455147520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353861584754263, + "loss": 2.6472, + "theoretical_loss": 3.954949132879154, + "tokens_seen": 455213056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353761283851555, + "loss": 2.7814, + "theoretical_loss": 3.9548868156946777, + "tokens_seen": 455278592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353660982948847, + "loss": 2.9024, + "theoretical_loss": 3.9548245099912216, + "tokens_seen": 455344128 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043535606820461386, + "loss": 2.7629, + "theoretical_loss": 3.954762215765019, + "tokens_seen": 455409664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 263055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.755061388015747, + "objective/train/theoretical_loss": 3.9546999330123036, + "objective/train/tokens_used": 475935200, + "theoretical_loss": 3.9546999330123036, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043534603811434304, + "loss": 2.8091, + "theoretical_loss": 3.9546999330123036, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043533600802407227, + "loss": 2.7972, + "theoretical_loss": 3.954637661729312, + "tokens_seen": 455540736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353259779338014, + "loss": 2.7967, + "theoretical_loss": 3.954575401912284, + "tokens_seen": 455606272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043531594784353063, + "loss": 2.8888, + "theoretical_loss": 3.954513153557458, + "tokens_seen": 455671808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043530591775325976, + "loss": 2.7848, + "theoretical_loss": 3.9544509166610755, + "tokens_seen": 455737344 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435295887662989, + "loss": 2.7125, + "theoretical_loss": 3.95438869121938, + "tokens_seen": 455802880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352858575727182, + "loss": 2.8886, + "theoretical_loss": 3.9543264772286184, + "tokens_seen": 455868416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043527582748244736, + "loss": 2.7964, + "theoretical_loss": 3.954264274685036, + "tokens_seen": 455933952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043526579739217654, + "loss": 3.0038, + "theoretical_loss": 3.954202083584882, + "tokens_seen": 455999488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352557673019057, + "loss": 3.0308, + "theoretical_loss": 3.954139903924407, + "tokens_seen": 456065024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352457372116349, + "loss": 2.8824, + "theoretical_loss": 3.9540777356998635, + "tokens_seen": 456130560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043523570712136414, + "loss": 2.8996, + "theoretical_loss": 3.954015578907506, + "tokens_seen": 456196096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043522567703109326, + "loss": 2.8487, + "theoretical_loss": 3.953953433543589, + "tokens_seen": 456261632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352156469408225, + "loss": 2.7798, + "theoretical_loss": 3.953891299604371, + "tokens_seen": 456327168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352056168505517, + "loss": 2.8852, + "theoretical_loss": 3.9538291770861123, + "tokens_seen": 456392704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043519558676028086, + "loss": 3.0216, + "theoretical_loss": 3.9537670659850725, + "tokens_seen": 456458240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043518555667001004, + "loss": 2.8016, + "theoretical_loss": 3.953704966297515, + "tokens_seen": 456523776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351755265797392, + "loss": 2.7142, + "theoretical_loss": 3.953642878019705, + "tokens_seen": 456589312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351654964894684, + "loss": 2.846, + "theoretical_loss": 3.953580801147908, + "tokens_seen": 456654848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043515546639919764, + "loss": 2.9923, + "theoretical_loss": 3.9535187356783927, + "tokens_seen": 456720384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043514543630892677, + "loss": 2.7513, + "theoretical_loss": 3.9534566816074292, + "tokens_seen": 456785920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435135406218656, + "loss": 2.7321, + "theoretical_loss": 3.953394638931289, + "tokens_seen": 456851456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043512537612838513, + "loss": 2.8101, + "theoretical_loss": 3.953332607646246, + "tokens_seen": 456916992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043511534603811437, + "loss": 2.754, + "theoretical_loss": 3.953270587748574, + "tokens_seen": 456982528 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043510531594784355, + "loss": 2.8689, + "theoretical_loss": 3.9532085792345515, + "tokens_seen": 457048064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 264378, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8818917274475098, + "objective/train/theoretical_loss": 3.953146582100456, + "objective/train/tokens_used": 477573600, + "theoretical_loss": 3.953146582100456, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043509528585757273, + "loss": 2.9812, + "theoretical_loss": 3.953146582100456, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350852557673019, + "loss": 2.7279, + "theoretical_loss": 3.9530845963425696, + "tokens_seen": 457179136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350752256770311, + "loss": 2.69, + "theoretical_loss": 3.9530226219571727, + "tokens_seen": 457244672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043506519558676027, + "loss": 2.698, + "theoretical_loss": 3.9529606589405497, + "tokens_seen": 457310208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350551654964895, + "loss": 2.8551, + "theoretical_loss": 3.952898707288987, + "tokens_seen": 457375744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043504513540621863, + "loss": 2.8347, + "theoretical_loss": 3.9528367669987716, + "tokens_seen": 457441280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043503510531594787, + "loss": 2.828, + "theoretical_loss": 3.952774838066193, + "tokens_seen": 457506816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043502507522567705, + "loss": 2.8248, + "theoretical_loss": 3.9527129204875413, + "tokens_seen": 457572352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043501504513540623, + "loss": 3.1028, + "theoretical_loss": 3.95265101425911, + "tokens_seen": 457637888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350050150451354, + "loss": 3.2986, + "theoretical_loss": 3.952589119377193, + "tokens_seen": 457703424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349949849548646, + "loss": 2.7862, + "theoretical_loss": 3.9525272358380867, + "tokens_seen": 457768960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349849548645938, + "loss": 2.91, + "theoretical_loss": 3.9524653636380895, + "tokens_seen": 457834496 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434974924774323, + "loss": 2.9778, + "theoretical_loss": 3.9524035027734996, + "tokens_seen": 457900032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043496489468405214, + "loss": 2.9951, + "theoretical_loss": 3.9523416532406204, + "tokens_seen": 457965568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043495486459378137, + "loss": 2.9673, + "theoretical_loss": 3.9522798150357525, + "tokens_seen": 458031104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349448345035105, + "loss": 2.7049, + "theoretical_loss": 3.952217988155203, + "tokens_seen": 458096640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043493480441323973, + "loss": 2.7651, + "theoretical_loss": 3.952156172595277, + "tokens_seen": 458162176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349247743229689, + "loss": 2.9111, + "theoretical_loss": 3.952094368352284, + "tokens_seen": 458227712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349147442326981, + "loss": 3.0533, + "theoretical_loss": 3.9520325754225327, + "tokens_seen": 458293248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349047141424273, + "loss": 2.8072, + "theoretical_loss": 3.951970793802336, + "tokens_seen": 458358784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348946840521565, + "loss": 2.4948, + "theoretical_loss": 3.9519090234880068, + "tokens_seen": 458424320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043488465396188564, + "loss": 2.8712, + "theoretical_loss": 3.9518472644758607, + "tokens_seen": 458489856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348746238716149, + "loss": 2.9302, + "theoretical_loss": 3.951785516762214, + "tokens_seen": 458555392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434864593781344, + "loss": 2.7907, + "theoretical_loss": 3.951723780343386, + "tokens_seen": 458620928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043485456369107324, + "loss": 2.9902, + "theoretical_loss": 3.951662055215697, + "tokens_seen": 458686464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 264937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.118872880935669, + "objective/train/theoretical_loss": 3.951600341375469, + "objective/train/tokens_used": 479212000, + "theoretical_loss": 3.951600341375469, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348445336008024, + "loss": 3.0182, + "theoretical_loss": 3.951600341375469, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348345035105316, + "loss": 2.8335, + "theoretical_loss": 3.9515386388190255, + "tokens_seen": 458817536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348244734202608, + "loss": 2.8455, + "theoretical_loss": 3.9514769475426927, + "tokens_seen": 458883072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043481444332998996, + "loss": 2.9251, + "theoretical_loss": 3.9514152675427976, + "tokens_seen": 458948608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480441323971914, + "loss": 2.7918, + "theoretical_loss": 3.9513535988156696, + "tokens_seen": 459014144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347943831494484, + "loss": 2.7582, + "theoretical_loss": 3.9512919413576384, + "tokens_seen": 459079680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347843530591775, + "loss": 2.9296, + "theoretical_loss": 3.951230295165038, + "tokens_seen": 459145216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043477432296890674, + "loss": 2.8319, + "theoretical_loss": 3.951168660234201, + "tokens_seen": 459210752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476429287863587, + "loss": 2.6616, + "theoretical_loss": 3.9511070365614644, + "tokens_seen": 459276288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347542627883651, + "loss": 2.8487, + "theoretical_loss": 3.951045424143165, + "tokens_seen": 459341824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347442326980943, + "loss": 2.7885, + "theoretical_loss": 3.9509838229756427, + "tokens_seen": 459407360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043473420260782346, + "loss": 2.9401, + "theoretical_loss": 3.950922233055238, + "tokens_seen": 459472896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472417251755265, + "loss": 2.8083, + "theoretical_loss": 3.9508606543782943, + "tokens_seen": 459538432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347141424272819, + "loss": 2.8683, + "theoretical_loss": 3.9507990869411556, + "tokens_seen": 459603968 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434704112337011, + "loss": 2.7853, + "theoretical_loss": 3.9507375307401684, + "tokens_seen": 459669504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043469408224674024, + "loss": 3.0062, + "theoretical_loss": 3.9506759857716798, + "tokens_seen": 459735040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346840521564694, + "loss": 2.7083, + "theoretical_loss": 3.95061445203204, + "tokens_seen": 459800576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346740220661986, + "loss": 3.0644, + "theoretical_loss": 3.9505529295176, + "tokens_seen": 459866112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043466399197592784, + "loss": 2.6074, + "theoretical_loss": 3.950491418224713, + "tokens_seen": 459931648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043465396188565697, + "loss": 2.8359, + "theoretical_loss": 3.9504299181497338, + "tokens_seen": 459997184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346439317953862, + "loss": 2.6802, + "theoretical_loss": 3.9503684292890187, + "tokens_seen": 460062720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043463390170511533, + "loss": 3.035, + "theoretical_loss": 3.9503069516389253, + "tokens_seen": 460128256 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043462387161484457, + "loss": 2.9591, + "theoretical_loss": 3.950245485195814, + "tokens_seen": 460193792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043461384152457375, + "loss": 2.8343, + "theoretical_loss": 3.9501840299560467, + "tokens_seen": 460259328 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043460381143430293, + "loss": 2.5444, + "theoretical_loss": 3.950122585915985, + "tokens_seen": 460324864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 265992, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.820603370666504, + "objective/train/theoretical_loss": 3.9500611530719953, + "objective/train/tokens_used": 480850400, + "theoretical_loss": 3.9500611530719953, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345937813440321, + "loss": 2.7728, + "theoretical_loss": 3.9500611530719953, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345837512537613, + "loss": 2.894, + "theoretical_loss": 3.9499997314204434, + "tokens_seen": 460455936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043457372116349047, + "loss": 2.8096, + "theoretical_loss": 3.949938320957698, + "tokens_seen": 460521472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345636910732197, + "loss": 2.4647, + "theoretical_loss": 3.949876921680129, + "tokens_seen": 460587008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043455366098294883, + "loss": 3.0128, + "theoretical_loss": 3.9498155335841076, + "tokens_seen": 460652544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043454363089267807, + "loss": 2.9574, + "theoretical_loss": 3.949754156666008, + "tokens_seen": 460718080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043453360080240725, + "loss": 2.8107, + "theoretical_loss": 3.949692790922205, + "tokens_seen": 460783616 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043452357071213643, + "loss": 2.7675, + "theoretical_loss": 3.9496314363490743, + "tokens_seen": 460849152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345135406218656, + "loss": 2.7283, + "theoretical_loss": 3.949570092942996, + "tokens_seen": 460914688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345035105315948, + "loss": 2.7483, + "theoretical_loss": 3.949508760700349, + "tokens_seen": 460980224 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434493480441324, + "loss": 2.7427, + "theoretical_loss": 3.949447439617516, + "tokens_seen": 461045760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344834503510532, + "loss": 2.9061, + "theoretical_loss": 3.94938612969088, + "tokens_seen": 461111296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043447342026078234, + "loss": 2.9127, + "theoretical_loss": 3.949324830916826, + "tokens_seen": 461176832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043446339017051157, + "loss": 2.9454, + "theoretical_loss": 3.949263543291741, + "tokens_seen": 461242368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344533600802407, + "loss": 2.848, + "theoretical_loss": 3.949202266812014, + "tokens_seen": 461307904 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043444332998996993, + "loss": 3.0778, + "theoretical_loss": 3.9491410014740347, + "tokens_seen": 461373440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344332998996991, + "loss": 2.9834, + "theoretical_loss": 3.949079747274195, + "tokens_seen": 461438976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344232698094283, + "loss": 2.8894, + "theoretical_loss": 3.9490185042088894, + "tokens_seen": 461504512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344132397191575, + "loss": 2.653, + "theoretical_loss": 3.948957272274512, + "tokens_seen": 461570048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344032096288867, + "loss": 2.8366, + "theoretical_loss": 3.94889605146746, + "tokens_seen": 461635584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043439317953861584, + "loss": 2.9733, + "theoretical_loss": 3.948834841784133, + "tokens_seen": 461701120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343831494483451, + "loss": 2.7639, + "theoretical_loss": 3.9487736432209304, + "tokens_seen": 461766656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343731193580742, + "loss": 3.0074, + "theoretical_loss": 3.9487124557742543, + "tokens_seen": 461832192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043436308926780344, + "loss": 3.0052, + "theoretical_loss": 3.948651279440509, + "tokens_seen": 461897728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343530591775326, + "loss": 3.129, + "theoretical_loss": 3.948590114216098, + "tokens_seen": 461963264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 266575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.696946144104004, + "objective/train/theoretical_loss": 3.9485289600974305, + "objective/train/tokens_used": 482488800, + "theoretical_loss": 3.9485289600974305, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343430290872618, + "loss": 2.6741, + "theoretical_loss": 3.9485289600974305, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434332998996991, + "loss": 2.8477, + "theoretical_loss": 3.948467817080915, + "tokens_seen": 462094336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043432296890672016, + "loss": 3.0804, + "theoretical_loss": 3.94840668516296, + "tokens_seen": 462159872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043431293881644934, + "loss": 2.9808, + "theoretical_loss": 3.9483455643399794, + "tokens_seen": 462225408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343029087261786, + "loss": 3.0524, + "theoretical_loss": 3.9482844546083857, + "tokens_seen": 462290944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342928786359077, + "loss": 2.8848, + "theoretical_loss": 3.948223355964595, + "tokens_seen": 462356480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428284854563694, + "loss": 2.9191, + "theoretical_loss": 3.948162268405024, + "tokens_seen": 462422016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043427281845536607, + "loss": 2.7022, + "theoretical_loss": 3.9481011919260913, + "tokens_seen": 462487552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342627883650953, + "loss": 2.759, + "theoretical_loss": 3.9480401265242175, + "tokens_seen": 462553088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342527582748245, + "loss": 2.7993, + "theoretical_loss": 3.9479790721958246, + "tokens_seen": 462618624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043424272818455367, + "loss": 2.7651, + "theoretical_loss": 3.9479180289373366, + "tokens_seen": 462684160 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043423269809428285, + "loss": 2.8418, + "theoretical_loss": 3.9478569967451778, + "tokens_seen": 462749696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342226680040121, + "loss": 2.7632, + "theoretical_loss": 3.947795975615776, + "tokens_seen": 462815232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342126379137412, + "loss": 2.9218, + "theoretical_loss": 3.9477349655455596, + "tokens_seen": 462880768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043420260782347044, + "loss": 2.9163, + "theoretical_loss": 3.9476739665309593, + "tokens_seen": 462946304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043419257773319957, + "loss": 2.6965, + "theoretical_loss": 3.947612978568406, + "tokens_seen": 463011840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341825476429288, + "loss": 2.9593, + "theoretical_loss": 3.9475520016543353, + "tokens_seen": 463077376 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434172517552658, + "loss": 2.9816, + "theoretical_loss": 3.9474910357851805, + "tokens_seen": 463142912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043416248746238717, + "loss": 2.9265, + "theoretical_loss": 3.9474300809573797, + "tokens_seen": 463208448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043415245737211635, + "loss": 2.7887, + "theoretical_loss": 3.947369137167371, + "tokens_seen": 463273984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043414242728184553, + "loss": 2.7265, + "theoretical_loss": 3.947308204411595, + "tokens_seen": 463339520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341323971915747, + "loss": 2.8877, + "theoretical_loss": 3.947247282686493, + "tokens_seen": 463405056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043412236710130395, + "loss": 2.8725, + "theoretical_loss": 3.947186371988509, + "tokens_seen": 463470592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341123370110331, + "loss": 2.7452, + "theoretical_loss": 3.947125472314088, + "tokens_seen": 463536128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341023069207623, + "loss": 2.7315, + "theoretical_loss": 3.947064583659678, + "tokens_seen": 463601664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 267814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8467938899993896, + "objective/train/theoretical_loss": 3.9470037060217256, + "objective/train/tokens_used": 484127200, + "theoretical_loss": 3.9470037060217256, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043409227683049144, + "loss": 2.9119, + "theoretical_loss": 3.9470037060217256, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043408224674022067, + "loss": 2.7476, + "theoretical_loss": 3.946942839396682, + "tokens_seen": 463732736 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043407221664994985, + "loss": 2.8435, + "theoretical_loss": 3.946881983780999, + "tokens_seen": 463798272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043406218655967903, + "loss": 3.0763, + "theoretical_loss": 3.9468211391711296, + "tokens_seen": 463863808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340521564694082, + "loss": 2.8394, + "theoretical_loss": 3.9467603055635294, + "tokens_seen": 463929344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043404212637913745, + "loss": 2.9131, + "theoretical_loss": 3.9466994829546542, + "tokens_seen": 463994880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340320962888666, + "loss": 3.0057, + "theoretical_loss": 3.946638671340964, + "tokens_seen": 464060416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340220661985958, + "loss": 2.8754, + "theoretical_loss": 3.946577870718917, + "tokens_seen": 464125952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043401203610832494, + "loss": 3.0896, + "theoretical_loss": 3.9465170810849757, + "tokens_seen": 464191488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340020060180542, + "loss": 2.7866, + "theoretical_loss": 3.946456302435603, + "tokens_seen": 464257024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043399197592778336, + "loss": 2.8963, + "theoretical_loss": 3.946395534767265, + "tokens_seen": 464322560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043398194583751254, + "loss": 2.9696, + "theoretical_loss": 3.946334778076427, + "tokens_seen": 464388096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339719157472417, + "loss": 2.8131, + "theoretical_loss": 3.9462740323595575, + "tokens_seen": 464453632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339618856569709, + "loss": 2.8574, + "theoretical_loss": 3.9462132976131263, + "tokens_seen": 464519168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339518555667001, + "loss": 2.7606, + "theoretical_loss": 3.946152573833605, + "tokens_seen": 464584704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339418254764293, + "loss": 3.0352, + "theoretical_loss": 3.9460918610174662, + "tokens_seen": 464650240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339317953861585, + "loss": 2.7562, + "theoretical_loss": 3.946031159161185, + "tokens_seen": 464715776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339217652958877, + "loss": 2.8604, + "theoretical_loss": 3.9459704682612387, + "tokens_seen": 464781312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339117352056169, + "loss": 2.8481, + "theoretical_loss": 3.945909788314104, + "tokens_seen": 464846848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043390170511534604, + "loss": 2.7717, + "theoretical_loss": 3.945849119316261, + "tokens_seen": 464912384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338916750250753, + "loss": 2.7045, + "theoretical_loss": 3.9457884612641902, + "tokens_seen": 464977920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338816449348044, + "loss": 2.9895, + "theoretical_loss": 3.9457278141543752, + "tokens_seen": 465043456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043387161484453364, + "loss": 2.8658, + "theoretical_loss": 3.9456671779833004, + "tokens_seen": 465108992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338615847542628, + "loss": 2.9028, + "theoretical_loss": 3.945606552747452, + "tokens_seen": 465174528 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433851554663992, + "loss": 2.954, + "theoretical_loss": 3.945545938443318, + "tokens_seen": 465240064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 268427, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.444042682647705, + "objective/train/theoretical_loss": 3.945485335067386, + "objective/train/tokens_used": 485765600, + "theoretical_loss": 3.945485335067386, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338415245737212, + "loss": 2.6741, + "theoretical_loss": 3.945485335067386, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043383149448345036, + "loss": 2.9785, + "theoretical_loss": 3.945424742616149, + "tokens_seen": 465371136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043382146439317954, + "loss": 2.8609, + "theoretical_loss": 3.945364161086099, + "tokens_seen": 465436672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338114343029088, + "loss": 2.82, + "theoretical_loss": 3.9453035904737304, + "tokens_seen": 465502208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338014042126379, + "loss": 2.964, + "theoretical_loss": 3.9452430307755386, + "tokens_seen": 465567744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043379137412236714, + "loss": 2.9767, + "theoretical_loss": 3.945182481988021, + "tokens_seen": 465633280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043378134403209627, + "loss": 2.921, + "theoretical_loss": 3.945121944107677, + "tokens_seen": 465698816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337713139418255, + "loss": 2.8756, + "theoretical_loss": 3.945061417131007, + "tokens_seen": 465764352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337612838515547, + "loss": 2.721, + "theoretical_loss": 3.945000901054514, + "tokens_seen": 465829888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043375125376128387, + "loss": 2.9334, + "theoretical_loss": 3.944940395874701, + "tokens_seen": 465895424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043374122367101305, + "loss": 2.9868, + "theoretical_loss": 3.944879901588074, + "tokens_seen": 465960960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337311935807423, + "loss": 2.7399, + "theoretical_loss": 3.9448194181911402, + "tokens_seen": 466026496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337211634904714, + "loss": 3.0384, + "theoretical_loss": 3.944758945680408, + "tokens_seen": 466092032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043371113340020064, + "loss": 2.8814, + "theoretical_loss": 3.9446984840523873, + "tokens_seen": 466157568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043370110330992977, + "loss": 2.6462, + "theoretical_loss": 3.9446380333035918, + "tokens_seen": 466223104 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433691073219659, + "loss": 2.6692, + "theoretical_loss": 3.944577593430534, + "tokens_seen": 466288640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336810431293882, + "loss": 2.8248, + "theoretical_loss": 3.9445171644297288, + "tokens_seen": 466354176 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043367101303911737, + "loss": 2.6765, + "theoretical_loss": 3.9444567462976936, + "tokens_seen": 466419712 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043366098294884655, + "loss": 2.8589, + "theoretical_loss": 3.9443963390309458, + "tokens_seen": 466485248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043365095285857573, + "loss": 2.9641, + "theoretical_loss": 3.9443359426260063, + "tokens_seen": 466550784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336409227683049, + "loss": 2.8887, + "theoretical_loss": 3.944275557079397, + "tokens_seen": 466616320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043363089267803415, + "loss": 2.9296, + "theoretical_loss": 3.9442151823876404, + "tokens_seen": 466681856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336208625877633, + "loss": 2.7081, + "theoretical_loss": 3.9441548185472612, + "tokens_seen": 466747392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336108324974925, + "loss": 2.7055, + "theoretical_loss": 3.9440944655547865, + "tokens_seen": 466812928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043360080240722164, + "loss": 3.0324, + "theoretical_loss": 3.944034123406744, + "tokens_seen": 466878464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 269429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1784250736236572, + "objective/train/theoretical_loss": 3.943973792099663, + "objective/train/tokens_used": 487404000, + "theoretical_loss": 3.943973792099663, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043359077231695087, + "loss": 2.9881, + "theoretical_loss": 3.943973792099663, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043358074222668005, + "loss": 2.8874, + "theoretical_loss": 3.943913471630075, + "tokens_seen": 467009536 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043357071213640923, + "loss": 2.8577, + "theoretical_loss": 3.943853161994513, + "tokens_seen": 467075072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335606820461384, + "loss": 2.9665, + "theoretical_loss": 3.943792863189511, + "tokens_seen": 467140608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043355065195586765, + "loss": 2.8564, + "theoretical_loss": 3.9437325752116053, + "tokens_seen": 467206144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335406218655968, + "loss": 2.8001, + "theoretical_loss": 3.943672298057333, + "tokens_seen": 467271680 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433530591775326, + "loss": 3.0148, + "theoretical_loss": 3.9436120317232337, + "tokens_seen": 467337216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043352056168505514, + "loss": 2.9508, + "theoretical_loss": 3.9435517762058483, + "tokens_seen": 467402752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335105315947844, + "loss": 2.9171, + "theoretical_loss": 3.9434915315017185, + "tokens_seen": 467468288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043350050150451356, + "loss": 2.7347, + "theoretical_loss": 3.943431297607389, + "tokens_seen": 467533824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043349047141424274, + "loss": 2.8175, + "theoretical_loss": 3.943371074519405, + "tokens_seen": 467599360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334804413239719, + "loss": 2.5834, + "theoretical_loss": 3.943310862234313, + "tokens_seen": 467664896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334704112337011, + "loss": 2.8371, + "theoretical_loss": 3.9432506607486633, + "tokens_seen": 467730432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334603811434303, + "loss": 2.6997, + "theoretical_loss": 3.9431904700590046, + "tokens_seen": 467795968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334503510531595, + "loss": 2.9174, + "theoretical_loss": 3.9431302901618897, + "tokens_seen": 467861504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043344032096288864, + "loss": 2.9733, + "theoretical_loss": 3.943070121053872, + "tokens_seen": 467927040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334302908726179, + "loss": 3.0873, + "theoretical_loss": 3.943009962731506, + "tokens_seen": 467992576 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433420260782347, + "loss": 2.8206, + "theoretical_loss": 3.9429498151913487, + "tokens_seen": 468058112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043341023069207624, + "loss": 2.6554, + "theoretical_loss": 3.9428896784299585, + "tokens_seen": 468123648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334002006018054, + "loss": 2.8164, + "theoretical_loss": 3.9428295524438948, + "tokens_seen": 468189184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333901705115346, + "loss": 2.8438, + "theoretical_loss": 3.9427694372297193, + "tokens_seen": 468254720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333801404212638, + "loss": 2.8599, + "theoretical_loss": 3.942709332783995, + "tokens_seen": 468320256 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433370110330993, + "loss": 2.7072, + "theoretical_loss": 3.942649239103286, + "tokens_seen": 468385792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043336008024072215, + "loss": 2.7251, + "theoretical_loss": 3.942589156184159, + "tokens_seen": 468451328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333500501504514, + "loss": 2.8781, + "theoretical_loss": 3.942529084023181, + "tokens_seen": 468516864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 269989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7550222873687744, + "objective/train/theoretical_loss": 3.9424690226169212, + "objective/train/tokens_used": 489042400, + "theoretical_loss": 3.9424690226169212, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333400200601805, + "loss": 2.889, + "theoretical_loss": 3.9424690226169212, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043332998996990974, + "loss": 2.9478, + "theoretical_loss": 3.9424089719619513, + "tokens_seen": 468647936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333199598796389, + "loss": 2.7374, + "theoretical_loss": 3.942348932054843, + "tokens_seen": 468713472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333099297893681, + "loss": 2.8397, + "theoretical_loss": 3.942288902892171, + "tokens_seen": 468779008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332998996990973, + "loss": 2.8605, + "theoretical_loss": 3.94222888447051, + "tokens_seen": 468844544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043328986960882647, + "loss": 2.8271, + "theoretical_loss": 3.942168876786438, + "tokens_seen": 468910080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327983951855565, + "loss": 2.7673, + "theoretical_loss": 3.942108879836532, + "tokens_seen": 468975616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332698094282849, + "loss": 2.8577, + "theoretical_loss": 3.942048893617374, + "tokens_seen": 469041152 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433259779338014, + "loss": 2.8374, + "theoretical_loss": 3.9419889181255456, + "tokens_seen": 469106688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043324974924774325, + "loss": 2.8189, + "theoretical_loss": 3.941928953357629, + "tokens_seen": 469172224 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323971915747243, + "loss": 2.5255, + "theoretical_loss": 3.941868999310211, + "tokens_seen": 469237760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332296890672016, + "loss": 2.6435, + "theoretical_loss": 3.9418090559798764, + "tokens_seen": 469303296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332196589769308, + "loss": 2.8742, + "theoretical_loss": 3.941749123363214, + "tokens_seen": 469368832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043320962888665997, + "loss": 2.7471, + "theoretical_loss": 3.9416892014568132, + "tokens_seen": 469434368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043319959879638915, + "loss": 2.8973, + "theoretical_loss": 3.941629290257266, + "tokens_seen": 469499904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331895687061184, + "loss": 2.7402, + "theoretical_loss": 3.941569389761164, + "tokens_seen": 469565440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043317953861584757, + "loss": 2.7395, + "theoretical_loss": 3.941509499965102, + "tokens_seen": 469630976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043316950852557675, + "loss": 2.9798, + "theoretical_loss": 3.9414496208656766, + "tokens_seen": 469696512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043315947843530593, + "loss": 2.8967, + "theoretical_loss": 3.941389752459484, + "tokens_seen": 469762048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331494483450351, + "loss": 2.9411, + "theoretical_loss": 3.941329894743124, + "tokens_seen": 469827584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043313941825476435, + "loss": 2.9326, + "theoretical_loss": 3.9412700477131972, + "tokens_seen": 469893120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331293881644935, + "loss": 2.8993, + "theoretical_loss": 3.9412102113663057, + "tokens_seen": 469958656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331193580742227, + "loss": 2.8847, + "theoretical_loss": 3.941150385699052, + "tokens_seen": 470024192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043310932798395184, + "loss": 2.7375, + "theoretical_loss": 3.941090570708043, + "tokens_seen": 470089728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043309929789368107, + "loss": 3.0306, + "theoretical_loss": 3.9410307663898845, + "tokens_seen": 470155264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 271247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.501755475997925, + "objective/train/theoretical_loss": 3.9409709727411846, + "objective/train/tokens_used": 490680800, + "theoretical_loss": 3.9409709727411846, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043308926780341025, + "loss": 2.8002, + "theoretical_loss": 3.9409709727411846, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043307923771313943, + "loss": 2.8405, + "theoretical_loss": 3.940911189758554, + "tokens_seen": 470286336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330692076228686, + "loss": 2.6275, + "theoretical_loss": 3.940851417438604, + "tokens_seen": 470351872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043305917753259785, + "loss": 2.8477, + "theoretical_loss": 3.9407916557779465, + "tokens_seen": 470417408 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433049147442327, + "loss": 2.8173, + "theoretical_loss": 3.9407319047731972, + "tokens_seen": 470482944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330391173520562, + "loss": 2.7989, + "theoretical_loss": 3.940672164420972, + "tokens_seen": 470548480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043302908726178534, + "loss": 2.9784, + "theoretical_loss": 3.940612434717888, + "tokens_seen": 470614016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330190571715146, + "loss": 3.0426, + "theoretical_loss": 3.9405527156605644, + "tokens_seen": 470679552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043300902708124376, + "loss": 2.8404, + "theoretical_loss": 3.9404930072456223, + "tokens_seen": 470745088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043299899699097294, + "loss": 2.6866, + "theoretical_loss": 3.9404333094696837, + "tokens_seen": 470810624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329889669007021, + "loss": 2.8504, + "theoretical_loss": 3.940373622329372, + "tokens_seen": 470876160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329789368104313, + "loss": 2.8516, + "theoretical_loss": 3.940313945821314, + "tokens_seen": 470941696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329689067201605, + "loss": 2.9968, + "theoretical_loss": 3.9402542799421347, + "tokens_seen": 471007232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329588766298897, + "loss": 2.8706, + "theoretical_loss": 3.9401946246884636, + "tokens_seen": 471072768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043294884653961884, + "loss": 2.8943, + "theoretical_loss": 3.94013498005693, + "tokens_seen": 471138304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329388164493481, + "loss": 2.789, + "theoretical_loss": 3.9400753460441655, + "tokens_seen": 471203840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329287863590772, + "loss": 2.6528, + "theoretical_loss": 3.940015722646804, + "tokens_seen": 471269376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043291875626880644, + "loss": 2.8134, + "theoretical_loss": 3.9399561098614786, + "tokens_seen": 471334912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329087261785356, + "loss": 3.0425, + "theoretical_loss": 3.939896507684827, + "tokens_seen": 471400448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328986960882648, + "loss": 2.8502, + "theoretical_loss": 3.939836916113485, + "tokens_seen": 471465984 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432888665997994, + "loss": 2.7366, + "theoretical_loss": 3.939777335144094, + "tokens_seen": 471531520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328786359077232, + "loss": 2.9817, + "theoretical_loss": 3.9397177647732926, + "tokens_seen": 471597056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043286860581745235, + "loss": 2.9623, + "theoretical_loss": 3.9396582049977242, + "tokens_seen": 471662592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328585757271816, + "loss": 2.7969, + "theoretical_loss": 3.9395986558140317, + "tokens_seen": 471728128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328485456369107, + "loss": 2.6316, + "theoretical_loss": 3.939539117218861, + "tokens_seen": 471793664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 272462, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.128762722015381, + "objective/train/theoretical_loss": 3.9394795892088585, + "objective/train/tokens_used": 492319200, + "theoretical_loss": 3.9394795892088585, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043283851554663994, + "loss": 2.9032, + "theoretical_loss": 3.9394795892088585, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328284854563691, + "loss": 2.7612, + "theoretical_loss": 3.939420071780673, + "tokens_seen": 471924736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328184553660983, + "loss": 2.8278, + "theoretical_loss": 3.939360564930954, + "tokens_seen": 471990272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328084252758275, + "loss": 2.7656, + "theoretical_loss": 3.939301068656353, + "tokens_seen": 472055808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043279839518555667, + "loss": 2.8529, + "theoretical_loss": 3.939241582953523, + "tokens_seen": 472121344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043278836509528585, + "loss": 2.8641, + "theoretical_loss": 3.9391821078191183, + "tokens_seen": 472186880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327783350050151, + "loss": 2.9301, + "theoretical_loss": 3.939122643249795, + "tokens_seen": 472252416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327683049147442, + "loss": 2.794, + "theoretical_loss": 3.9390631892422103, + "tokens_seen": 472317952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043275827482447345, + "loss": 2.9563, + "theoretical_loss": 3.9390037457930234, + "tokens_seen": 472383488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043274824473420263, + "loss": 2.6417, + "theoretical_loss": 3.9389443128988946, + "tokens_seen": 472449024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327382146439318, + "loss": 2.9363, + "theoretical_loss": 3.938884890556486, + "tokens_seen": 472514560 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432728184553661, + "loss": 2.9786, + "theoretical_loss": 3.9388254787624617, + "tokens_seen": 472580096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043271815446339017, + "loss": 2.8273, + "theoretical_loss": 3.938766077513486, + "tokens_seen": 472645632 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043270812437311935, + "loss": 2.8662, + "theoretical_loss": 3.938706686806226, + "tokens_seen": 472711168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326980942828486, + "loss": 2.9246, + "theoretical_loss": 3.9386473066373497, + "tokens_seen": 472776704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326880641925777, + "loss": 2.9862, + "theoretical_loss": 3.9385879370035264, + "tokens_seen": 472842240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043267803410230695, + "loss": 2.6308, + "theoretical_loss": 3.938528577901427, + "tokens_seen": 472907776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326680040120361, + "loss": 2.9238, + "theoretical_loss": 3.9384692293277253, + "tokens_seen": 472973312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326579739217653, + "loss": 2.7366, + "theoretical_loss": 3.938409891279094, + "tokens_seen": 473038848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326479438314945, + "loss": 2.6296, + "theoretical_loss": 3.93835056375221, + "tokens_seen": 473104384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326379137412237, + "loss": 2.8136, + "theoretical_loss": 3.9382912467437503, + "tokens_seen": 473169920 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043262788365095286, + "loss": 2.6962, + "theoretical_loss": 3.938231940250393, + "tokens_seen": 473235456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043261785356068204, + "loss": 2.8113, + "theoretical_loss": 3.9381726442688185, + "tokens_seen": 473300992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326078234704112, + "loss": 2.842, + "theoretical_loss": 3.938113358795709, + "tokens_seen": 473366528 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043259779338014045, + "loss": 2.8583, + "theoretical_loss": 3.9380540838277467, + "tokens_seen": 473432064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 273042, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9048469066619873, + "objective/train/theoretical_loss": 3.9379948193616174, + "objective/train/tokens_used": 493957600, + "theoretical_loss": 3.9379948193616174, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325877632898696, + "loss": 2.6338, + "theoretical_loss": 3.9379948193616174, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325777331995988, + "loss": 2.8322, + "theoretical_loss": 3.9379355653940062, + "tokens_seen": 473563136 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432567703109328, + "loss": 2.8898, + "theoretical_loss": 3.9378763219216015, + "tokens_seen": 473628672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325576730190572, + "loss": 2.6278, + "theoretical_loss": 3.937817088941093, + "tokens_seen": 473694208 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043254764292878636, + "loss": 2.7849, + "theoretical_loss": 3.9377578664491706, + "tokens_seen": 473759744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043253761283851554, + "loss": 2.7501, + "theoretical_loss": 3.937698654442527, + "tokens_seen": 473825280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325275827482447, + "loss": 2.8944, + "theoretical_loss": 3.937639452917856, + "tokens_seen": 473890816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043251755265797396, + "loss": 3.0332, + "theoretical_loss": 3.9375802618718523, + "tokens_seen": 473956352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325075225677031, + "loss": 2.9932, + "theoretical_loss": 3.9375210813012127, + "tokens_seen": 474021888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324974924774323, + "loss": 2.9509, + "theoretical_loss": 3.937461911202636, + "tokens_seen": 474087424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043248746238716145, + "loss": 2.8843, + "theoretical_loss": 3.9374027515728214, + "tokens_seen": 474152960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324774322968907, + "loss": 3.1177, + "theoretical_loss": 3.93734360240847, + "tokens_seen": 474218496 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043246740220661986, + "loss": 2.8071, + "theoretical_loss": 3.937284463706286, + "tokens_seen": 474284032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043245737211634904, + "loss": 2.8188, + "theoretical_loss": 3.937225335462971, + "tokens_seen": 474349568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324473420260782, + "loss": 2.8953, + "theoretical_loss": 3.937166217675233, + "tokens_seen": 474415104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324373119358074, + "loss": 2.719, + "theoretical_loss": 3.9371071103397783, + "tokens_seen": 474480640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043242728184553664, + "loss": 2.6903, + "theoretical_loss": 3.9370480134533157, + "tokens_seen": 474546176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324172517552658, + "loss": 2.7762, + "theoretical_loss": 3.9369889270125547, + "tokens_seen": 474611712 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432407221664995, + "loss": 2.5037, + "theoretical_loss": 3.9369298510142077, + "tokens_seen": 474677248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323971915747242, + "loss": 3.0455, + "theoretical_loss": 3.936870785454988, + "tokens_seen": 474742784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323871614844534, + "loss": 2.8792, + "theoretical_loss": 3.93681173033161, + "tokens_seen": 474808320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043237713139418255, + "loss": 2.7244, + "theoretical_loss": 3.9367526856407897, + "tokens_seen": 474873856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323671013039118, + "loss": 2.9051, + "theoretical_loss": 3.9366936513792448, + "tokens_seen": 474939392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323570712136409, + "loss": 2.7048, + "theoretical_loss": 3.936634627543694, + "tokens_seen": 475004928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043234704112337014, + "loss": 2.9354, + "theoretical_loss": 3.9365756141308594, + "tokens_seen": 475070464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 273516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.271859645843506, + "objective/train/theoretical_loss": 3.9365166111374608, + "objective/train/tokens_used": 495596000, + "theoretical_loss": 3.9365166111374608, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323370110330993, + "loss": 2.8993, + "theoretical_loss": 3.9365166111374608, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323269809428285, + "loss": 2.8789, + "theoretical_loss": 3.9364576185602234, + "tokens_seen": 475201536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323169508525577, + "loss": 2.9238, + "theoretical_loss": 3.9363986363958725, + "tokens_seen": 475267072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043230692076228687, + "loss": 2.7607, + "theoretical_loss": 3.936339664641133, + "tokens_seen": 475332608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043229689067201605, + "loss": 2.8522, + "theoretical_loss": 3.9362807032927343, + "tokens_seen": 475398144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322868605817453, + "loss": 2.9343, + "theoretical_loss": 3.9362217523474046, + "tokens_seen": 475463680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322768304914744, + "loss": 3.034, + "theoretical_loss": 3.9361628118018768, + "tokens_seen": 475529216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043226680040120365, + "loss": 2.9514, + "theoretical_loss": 3.936103881652881, + "tokens_seen": 475594752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043225677031093283, + "loss": 2.8471, + "theoretical_loss": 3.936044961897153, + "tokens_seen": 475660288 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432246740220662, + "loss": 2.8487, + "theoretical_loss": 3.9359860525314274, + "tokens_seen": 475725824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322367101303912, + "loss": 2.7933, + "theoretical_loss": 3.9359271535524414, + "tokens_seen": 475791360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043222668004012037, + "loss": 2.7604, + "theoretical_loss": 3.935868264956932, + "tokens_seen": 475856896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043221664994984955, + "loss": 2.8025, + "theoretical_loss": 3.9358093867416413, + "tokens_seen": 475922432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322066198595788, + "loss": 2.9206, + "theoretical_loss": 3.935750518903309, + "tokens_seen": 475987968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321965897693079, + "loss": 3.1348, + "theoretical_loss": 3.9356916614386783, + "tokens_seen": 476053504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043218655967903715, + "loss": 2.8289, + "theoretical_loss": 3.9356328143444923, + "tokens_seen": 476119040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321765295887663, + "loss": 2.77, + "theoretical_loss": 3.9355739776174987, + "tokens_seen": 476184576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321664994984955, + "loss": 2.9714, + "theoretical_loss": 3.935515151254444, + "tokens_seen": 476250112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321564694082247, + "loss": 3.0004, + "theoretical_loss": 3.935456335252076, + "tokens_seen": 476315648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321464393179539, + "loss": 2.7851, + "theoretical_loss": 3.9353975296071453, + "tokens_seen": 476381184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043213640922768306, + "loss": 2.9986, + "theoretical_loss": 3.9353387343164035, + "tokens_seen": 476446720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043212637913741224, + "loss": 2.8154, + "theoretical_loss": 3.9352799493766035, + "tokens_seen": 476512256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321163490471414, + "loss": 2.9255, + "theoretical_loss": 3.9352211747845, + "tokens_seen": 476577792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043210631895687065, + "loss": 2.792, + "theoretical_loss": 3.935162410536849, + "tokens_seen": 476643328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320962888665998, + "loss": 2.5768, + "theoretical_loss": 3.9351036566304076, + "tokens_seen": 476708864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 274597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.601290225982666, + "objective/train/theoretical_loss": 3.935044913061936, + "objective/train/tokens_used": 497234400, + "theoretical_loss": 3.935044913061936, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432086258776329, + "loss": 2.8793, + "theoretical_loss": 3.935044913061936, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320762286860582, + "loss": 2.7909, + "theoretical_loss": 3.9349861798281918, + "tokens_seen": 476839936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320661985957874, + "loss": 2.4871, + "theoretical_loss": 3.934927456925939, + "tokens_seen": 476905472 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043205616850551656, + "loss": 2.6598, + "theoretical_loss": 3.934868744351941, + "tokens_seen": 476971008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043204613841524574, + "loss": 2.6697, + "theoretical_loss": 3.934810042102961, + "tokens_seen": 477036544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320361083249749, + "loss": 2.8944, + "theoretical_loss": 3.934751350175766, + "tokens_seen": 477102080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043202607823470416, + "loss": 2.8022, + "theoretical_loss": 3.9346926685671235, + "tokens_seen": 477167616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320160481444333, + "loss": 2.9025, + "theoretical_loss": 3.934633997273803, + "tokens_seen": 477233152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320060180541625, + "loss": 2.8173, + "theoretical_loss": 3.934575336292575, + "tokens_seen": 477298688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043199598796389165, + "loss": 2.9114, + "theoretical_loss": 3.9345166856202107, + "tokens_seen": 477364224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319859578736209, + "loss": 2.9045, + "theoretical_loss": 3.9344580452534847, + "tokens_seen": 477429760 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043197592778335006, + "loss": 2.8471, + "theoretical_loss": 3.934399415189171, + "tokens_seen": 477495296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043196589769307924, + "loss": 2.6342, + "theoretical_loss": 3.9343407954240464, + "tokens_seen": 477560832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319558676028084, + "loss": 2.8732, + "theoretical_loss": 3.934282185954889, + "tokens_seen": 477626368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319458375125376, + "loss": 2.5875, + "theoretical_loss": 3.934223586778477, + "tokens_seen": 477691904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319358074222668, + "loss": 2.9511, + "theoretical_loss": 3.9341649978915925, + "tokens_seen": 477757440 + }, + { + "epoch": 0.14, + "learning_rate": 0.000431925777331996, + "loss": 2.8133, + "theoretical_loss": 3.9341064192910165, + "tokens_seen": 477822976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043191574724172515, + "loss": 2.9208, + "theoretical_loss": 3.9340478509735326, + "tokens_seen": 477888512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319057171514544, + "loss": 2.8644, + "theoretical_loss": 3.9339892929359266, + "tokens_seen": 477954048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043189568706118357, + "loss": 2.9012, + "theoretical_loss": 3.933930745174985, + "tokens_seen": 478019584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043188565697091275, + "loss": 2.5838, + "theoretical_loss": 3.9338722076874957, + "tokens_seen": 478085120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043187562688064193, + "loss": 2.7064, + "theoretical_loss": 3.933813680470247, + "tokens_seen": 478150656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318655967903711, + "loss": 2.9544, + "theoretical_loss": 3.9337551635200314, + "tokens_seen": 478216192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318555667001003, + "loss": 2.7427, + "theoretical_loss": 3.93369665683364, + "tokens_seen": 478281728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318455366098295, + "loss": 2.8325, + "theoretical_loss": 3.9336381604078667, + "tokens_seen": 478347264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 275931, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.969980239868164, + "objective/train/theoretical_loss": 3.933579674239507, + "objective/train/tokens_used": 498872800, + "theoretical_loss": 3.933579674239507, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183550651955865, + "loss": 2.9991, + "theoretical_loss": 3.933579674239507, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318254764292879, + "loss": 2.8388, + "theoretical_loss": 3.933521198325357, + "tokens_seen": 478478336 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431815446339017, + "loss": 2.872, + "theoretical_loss": 3.933462732662215, + "tokens_seen": 478543872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043180541624874625, + "loss": 2.9391, + "theoretical_loss": 3.933404277246881, + "tokens_seen": 478609408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043179538615847543, + "loss": 2.7832, + "theoretical_loss": 3.9333458320761547, + "tokens_seen": 478674944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317853560682046, + "loss": 2.8224, + "theoretical_loss": 3.93328739714684, + "tokens_seen": 478740480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317753259779338, + "loss": 2.6799, + "theoretical_loss": 3.9332289724557397, + "tokens_seen": 478806016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043176529588766303, + "loss": 2.6965, + "theoretical_loss": 3.9331705579996585, + "tokens_seen": 478871552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043175526579739215, + "loss": 2.9734, + "theoretical_loss": 3.9331121537754044, + "tokens_seen": 478937088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317452357071214, + "loss": 2.726, + "theoretical_loss": 3.933053759779784, + "tokens_seen": 479002624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317352056168505, + "loss": 2.5974, + "theoretical_loss": 3.9329953760096084, + "tokens_seen": 479068160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043172517552657975, + "loss": 2.7127, + "theoretical_loss": 3.9329370024616876, + "tokens_seen": 479133696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043171514543630893, + "loss": 2.7597, + "theoretical_loss": 3.9328786391328343, + "tokens_seen": 479199232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317051153460381, + "loss": 2.9107, + "theoretical_loss": 3.9328202860198616, + "tokens_seen": 479264768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316950852557673, + "loss": 2.9456, + "theoretical_loss": 3.9327619431195853, + "tokens_seen": 479330304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316850551654965, + "loss": 2.769, + "theoretical_loss": 3.9327036104288227, + "tokens_seen": 479395840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316750250752257, + "loss": 3.0487, + "theoretical_loss": 3.932645287944391, + "tokens_seen": 479461376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316649949849549, + "loss": 3.0331, + "theoretical_loss": 3.932586975663109, + "tokens_seen": 479526912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316549648946841, + "loss": 3.109, + "theoretical_loss": 3.9325286735817997, + "tokens_seen": 479592448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043164493480441326, + "loss": 2.7279, + "theoretical_loss": 3.9324703816972844, + "tokens_seen": 479657984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043163490471414244, + "loss": 2.7961, + "theoretical_loss": 3.9324121000063865, + "tokens_seen": 479723520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316248746238716, + "loss": 2.6693, + "theoretical_loss": 3.9323538285059314, + "tokens_seen": 479789056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043161484453360085, + "loss": 2.9624, + "theoretical_loss": 3.932295567192747, + "tokens_seen": 479854592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043160481444333, + "loss": 2.8733, + "theoretical_loss": 3.932237316063659, + "tokens_seen": 479920128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315947843530592, + "loss": 2.8734, + "theoretical_loss": 3.9321790751154984, + "tokens_seen": 479985664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 276348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0607144832611084, + "objective/train/theoretical_loss": 3.9321208443450963, + "objective/train/tokens_used": 500511200, + "theoretical_loss": 3.9321208443450963, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315847542627884, + "loss": 2.938, + "theoretical_loss": 3.9321208443450963, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315747241725176, + "loss": 2.5545, + "theoretical_loss": 3.9320626237492844, + "tokens_seen": 480116736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043156469408224676, + "loss": 2.6545, + "theoretical_loss": 3.9320044133248966, + "tokens_seen": 480182272 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043155466399197594, + "loss": 2.8061, + "theoretical_loss": 3.931946213068768, + "tokens_seen": 480247808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315446339017051, + "loss": 2.6674, + "theoretical_loss": 3.931888022977735, + "tokens_seen": 480313344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043153460381143436, + "loss": 2.9156, + "theoretical_loss": 3.9318298430486367, + "tokens_seen": 480378880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315245737211635, + "loss": 2.8293, + "theoretical_loss": 3.931771673278311, + "tokens_seen": 480444416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315145436308927, + "loss": 3.0808, + "theoretical_loss": 3.9317135136635994, + "tokens_seen": 480509952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043150451354062185, + "loss": 2.7806, + "theoretical_loss": 3.931655364201344, + "tokens_seen": 480575488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314944834503511, + "loss": 2.9279, + "theoretical_loss": 3.931597224888389, + "tokens_seen": 480641024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043148445336008026, + "loss": 3.0468, + "theoretical_loss": 3.931539095721578, + "tokens_seen": 480706560 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043147442326980944, + "loss": 3.0113, + "theoretical_loss": 3.9314809766977596, + "tokens_seen": 480772096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314643931795386, + "loss": 2.9067, + "theoretical_loss": 3.93142286781378, + "tokens_seen": 480837632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314543630892678, + "loss": 2.8236, + "theoretical_loss": 3.931364769066489, + "tokens_seen": 480903168 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431444332998997, + "loss": 3.1, + "theoretical_loss": 3.931306680452737, + "tokens_seen": 480968704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314343029087262, + "loss": 3.1625, + "theoretical_loss": 3.9312486019693766, + "tokens_seen": 481034240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043142427281845535, + "loss": 2.8898, + "theoretical_loss": 3.931190533613261, + "tokens_seen": 481099776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314142427281846, + "loss": 2.9089, + "theoretical_loss": 3.9311324753812453, + "tokens_seen": 481165312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043140421263791377, + "loss": 2.8168, + "theoretical_loss": 3.931074427270186, + "tokens_seen": 481230848 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043139418254764295, + "loss": 2.9268, + "theoretical_loss": 3.93101638927694, + "tokens_seen": 481296384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043138415245737213, + "loss": 2.8559, + "theoretical_loss": 3.9309583613983676, + "tokens_seen": 481361920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313741223671013, + "loss": 2.8534, + "theoretical_loss": 3.930900343631328, + "tokens_seen": 481427456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313640922768305, + "loss": 2.7964, + "theoretical_loss": 3.9308423359726845, + "tokens_seen": 481492992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313540621865597, + "loss": 2.8632, + "theoretical_loss": 3.930784338419299, + "tokens_seen": 481558528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043134403209628885, + "loss": 2.6951, + "theoretical_loss": 3.9307263509680372, + "tokens_seen": 481624064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 277290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.784728765487671, + "objective/train/theoretical_loss": 3.930668373615765, + "objective/train/tokens_used": 502149600, + "theoretical_loss": 3.930668373615765, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313340020060181, + "loss": 2.8966, + "theoretical_loss": 3.930668373615765, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313239719157472, + "loss": 3.0139, + "theoretical_loss": 3.9306104063593503, + "tokens_seen": 481755136 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043131394182547645, + "loss": 2.8475, + "theoretical_loss": 3.9305524491956616, + "tokens_seen": 481820672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043130391173520563, + "loss": 2.8799, + "theoretical_loss": 3.9304945021215687, + "tokens_seen": 481886208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312938816449348, + "loss": 2.8447, + "theoretical_loss": 3.9304365651339443, + "tokens_seen": 481951744 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431283851554664, + "loss": 3.0417, + "theoretical_loss": 3.9303786382296604, + "tokens_seen": 482017280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127382146439323, + "loss": 2.7033, + "theoretical_loss": 3.930320721405593, + "tokens_seen": 482082816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043126379137412236, + "loss": 2.8143, + "theoretical_loss": 3.9302628146586165, + "tokens_seen": 482148352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312537612838516, + "loss": 3.1353, + "theoretical_loss": 3.930204917985609, + "tokens_seen": 482213888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312437311935807, + "loss": 2.8771, + "theoretical_loss": 3.9301470313834495, + "tokens_seen": 482279424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043123370110330995, + "loss": 3.1197, + "theoretical_loss": 3.9300891548490178, + "tokens_seen": 482344960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043122367101303913, + "loss": 2.7898, + "theoretical_loss": 3.9300312883791944, + "tokens_seen": 482410496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312136409227683, + "loss": 2.7825, + "theoretical_loss": 3.929973431970863, + "tokens_seen": 482476032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312036108324975, + "loss": 3.0488, + "theoretical_loss": 3.9299155856209085, + "tokens_seen": 482541568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311935807422267, + "loss": 2.6776, + "theoretical_loss": 3.9298577493262155, + "tokens_seen": 482607104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043118355065195586, + "loss": 2.9107, + "theoretical_loss": 3.9297999230836713, + "tokens_seen": 482672640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311735205616851, + "loss": 2.9791, + "theoretical_loss": 3.9297421068901643, + "tokens_seen": 482738176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311634904714142, + "loss": 2.7728, + "theoretical_loss": 3.9296843007425846, + "tokens_seen": 482803712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043115346038114346, + "loss": 2.9737, + "theoretical_loss": 3.9296265046378225, + "tokens_seen": 482869248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311434302908726, + "loss": 2.8517, + "theoretical_loss": 3.9295687185727717, + "tokens_seen": 482934784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311334002006018, + "loss": 2.7383, + "theoretical_loss": 3.929510942544325, + "tokens_seen": 483000320 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431123370110331, + "loss": 3.0246, + "theoretical_loss": 3.9294531765493788, + "tokens_seen": 483065856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311133400200602, + "loss": 2.8883, + "theoretical_loss": 3.9293954205848296, + "tokens_seen": 483131392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043110330992978936, + "loss": 2.4493, + "theoretical_loss": 3.9293376746475746, + "tokens_seen": 483196928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310932798395186, + "loss": 3.0786, + "theoretical_loss": 3.9292799387345143, + "tokens_seen": 483262464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 277804, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7938942909240723, + "objective/train/theoretical_loss": 3.929222212842549, + "objective/train/tokens_used": 503788000, + "theoretical_loss": 3.929222212842549, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310832497492477, + "loss": 3.0089, + "theoretical_loss": 3.929222212842549, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043107321965897696, + "loss": 2.86, + "theoretical_loss": 3.929164496968581, + "tokens_seen": 483393536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310631895687061, + "loss": 2.8599, + "theoretical_loss": 3.929106791109514, + "tokens_seen": 483459072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310531594784353, + "loss": 3.1346, + "theoretical_loss": 3.9290490952622523, + "tokens_seen": 483524608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310431293881645, + "loss": 2.5788, + "theoretical_loss": 3.9289914094237037, + "tokens_seen": 483590144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310330992978937, + "loss": 3.0451, + "theoretical_loss": 3.9289337335907746, + "tokens_seen": 483655680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043102306920762286, + "loss": 2.8503, + "theoretical_loss": 3.928876067760375, + "tokens_seen": 483721216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043101303911735205, + "loss": 2.7226, + "theoretical_loss": 3.9288184119294147, + "tokens_seen": 483786752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310030090270812, + "loss": 2.841, + "theoretical_loss": 3.928760766094806, + "tokens_seen": 483852288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043099297893681046, + "loss": 2.9864, + "theoretical_loss": 3.928703130253462, + "tokens_seen": 483917824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309829488465396, + "loss": 2.8682, + "theoretical_loss": 3.9286455044022968, + "tokens_seen": 483983360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309729187562688, + "loss": 2.7043, + "theoretical_loss": 3.9285878885382273, + "tokens_seen": 484048896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043096288866599795, + "loss": 2.4423, + "theoretical_loss": 3.9285302826581696, + "tokens_seen": 484114432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309528585757272, + "loss": 2.6732, + "theoretical_loss": 3.9284726867590436, + "tokens_seen": 484179968 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043094282848545637, + "loss": 2.9397, + "theoretical_loss": 3.9284151008377686, + "tokens_seen": 484245504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043093279839518555, + "loss": 2.6799, + "theoretical_loss": 3.9283575248912666, + "tokens_seen": 484311040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309227683049148, + "loss": 2.4629, + "theoretical_loss": 3.92829995891646, + "tokens_seen": 484376576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043091273821464397, + "loss": 2.809, + "theoretical_loss": 3.928242402910273, + "tokens_seen": 484442112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043090270812437315, + "loss": 2.5356, + "theoretical_loss": 3.928184856869632, + "tokens_seen": 484507648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043089267803410233, + "loss": 2.8043, + "theoretical_loss": 3.9281273207914618, + "tokens_seen": 484573184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308826479438315, + "loss": 2.8214, + "theoretical_loss": 3.928069794672693, + "tokens_seen": 484638720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308726178535607, + "loss": 2.8554, + "theoretical_loss": 3.928012278510254, + "tokens_seen": 484704256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308625877632899, + "loss": 2.7812, + "theoretical_loss": 3.9279547723010753, + "tokens_seen": 484769792 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043085255767301905, + "loss": 3.1759, + "theoretical_loss": 3.9278972760420903, + "tokens_seen": 484835328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308425275827483, + "loss": 2.7076, + "theoretical_loss": 3.927839789730232, + "tokens_seen": 484900864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 279062, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1616718769073486, + "objective/train/theoretical_loss": 3.9277823133624366, + "objective/train/tokens_used": 505426400, + "theoretical_loss": 3.9277823133624366, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308324974924774, + "loss": 2.8253, + "theoretical_loss": 3.9277823133624366, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043082246740220665, + "loss": 2.749, + "theoretical_loss": 3.927724846935639, + "tokens_seen": 485031936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043081243731193583, + "loss": 2.7862, + "theoretical_loss": 3.9276673904467776, + "tokens_seen": 485097472 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430802407221665, + "loss": 2.8595, + "theoretical_loss": 3.927609943892792, + "tokens_seen": 485163008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307923771313942, + "loss": 2.7832, + "theoretical_loss": 3.9275525072706223, + "tokens_seen": 485228544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043078234704112343, + "loss": 2.7835, + "theoretical_loss": 3.9274950805772098, + "tokens_seen": 485294080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043077231695085256, + "loss": 2.7645, + "theoretical_loss": 3.9274376638094983, + "tokens_seen": 485359616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307622868605818, + "loss": 2.8211, + "theoretical_loss": 3.9273802569644323, + "tokens_seen": 485425152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307522567703109, + "loss": 2.6924, + "theoretical_loss": 3.9273228600389585, + "tokens_seen": 485490688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043074222668004015, + "loss": 2.7889, + "theoretical_loss": 3.9272654730300225, + "tokens_seen": 485556224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043073219658976933, + "loss": 3.0397, + "theoretical_loss": 3.9272080959345734, + "tokens_seen": 485621760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307221664994985, + "loss": 2.699, + "theoretical_loss": 3.927150728749562, + "tokens_seen": 485687296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307121364092277, + "loss": 2.729, + "theoretical_loss": 3.9270933714719396, + "tokens_seen": 485752832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307021063189569, + "loss": 2.7437, + "theoretical_loss": 3.9270360240986575, + "tokens_seen": 485818368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043069207622868606, + "loss": 2.8264, + "theoretical_loss": 3.926978686626671, + "tokens_seen": 485883904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306820461384153, + "loss": 2.9605, + "theoretical_loss": 3.926921359052935, + "tokens_seen": 485949440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306720160481444, + "loss": 3.0128, + "theoretical_loss": 3.9268640413744063, + "tokens_seen": 486014976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043066198595787366, + "loss": 2.7019, + "theoretical_loss": 3.9268067335880428, + "tokens_seen": 486080512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306519558676028, + "loss": 2.9512, + "theoretical_loss": 3.9267494356908044, + "tokens_seen": 486146048 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430641925777332, + "loss": 2.9346, + "theoretical_loss": 3.926692147679651, + "tokens_seen": 486211584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306318956870612, + "loss": 2.6555, + "theoretical_loss": 3.926634869551545, + "tokens_seen": 486277120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306218655967904, + "loss": 2.9892, + "theoretical_loss": 3.9265776013034497, + "tokens_seen": 486342656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043061183550651956, + "loss": 2.7261, + "theoretical_loss": 3.9265203429323305, + "tokens_seen": 486408192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306018054162488, + "loss": 2.6338, + "theoretical_loss": 3.926463094435153, + "tokens_seen": 486473728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305917753259779, + "loss": 2.7808, + "theoretical_loss": 3.9264058558088846, + "tokens_seen": 486539264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 279686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.470508337020874, + "objective/train/theoretical_loss": 3.926348627050494, + "objective/train/tokens_used": 507064800, + "theoretical_loss": 3.926348627050494, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043058174523570716, + "loss": 2.8973, + "theoretical_loss": 3.926348627050494, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305717151454363, + "loss": 2.9057, + "theoretical_loss": 3.9262914081569518, + "tokens_seen": 486670336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305616850551655, + "loss": 3.0043, + "theoretical_loss": 3.9262341991252296, + "tokens_seen": 486735872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305516549648947, + "loss": 2.6892, + "theoretical_loss": 3.9261769999522995, + "tokens_seen": 486801408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305416248746239, + "loss": 2.7105, + "theoretical_loss": 3.926119810635136, + "tokens_seen": 486866944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043053159478435306, + "loss": 3.0511, + "theoretical_loss": 3.9260626311707143, + "tokens_seen": 486932480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043052156469408225, + "loss": 2.6944, + "theoretical_loss": 3.926005461556011, + "tokens_seen": 486998016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305115346038114, + "loss": 2.5711, + "theoretical_loss": 3.9259483017880052, + "tokens_seen": 487063552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043050150451354066, + "loss": 2.9071, + "theoretical_loss": 3.9258911518636763, + "tokens_seen": 487129088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304914744232698, + "loss": 2.9919, + "theoretical_loss": 3.9258340117800037, + "tokens_seen": 487194624 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430481444332999, + "loss": 2.6231, + "theoretical_loss": 3.925776881533971, + "tokens_seen": 487260160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043047141424272815, + "loss": 3.1136, + "theoretical_loss": 3.9257197611225614, + "tokens_seen": 487325696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304613841524574, + "loss": 2.8313, + "theoretical_loss": 3.925662650542759, + "tokens_seen": 487391232 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043045135406218657, + "loss": 2.8077, + "theoretical_loss": 3.92560554979155, + "tokens_seen": 487456768 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043044132397191575, + "loss": 2.933, + "theoretical_loss": 3.925548458865923, + "tokens_seen": 487522304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043043129388164493, + "loss": 2.7352, + "theoretical_loss": 3.925491377762866, + "tokens_seen": 487587840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043042126379137417, + "loss": 2.9462, + "theoretical_loss": 3.925434306479369, + "tokens_seen": 487653376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304112337011033, + "loss": 2.8203, + "theoretical_loss": 3.925377245012424, + "tokens_seen": 487718912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043040120361083253, + "loss": 2.6397, + "theoretical_loss": 3.9253201933590227, + "tokens_seen": 487784448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043039117352056165, + "loss": 2.8775, + "theoretical_loss": 3.92526315151616, + "tokens_seen": 487849984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303811434302909, + "loss": 2.7412, + "theoretical_loss": 3.9252061194808316, + "tokens_seen": 487915520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043037111334002007, + "loss": 2.9406, + "theoretical_loss": 3.925149097250033, + "tokens_seen": 487981056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043036108324974925, + "loss": 2.9284, + "theoretical_loss": 3.925092084820764, + "tokens_seen": 488046592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043035105315947843, + "loss": 2.7134, + "theoretical_loss": 3.9250350821900226, + "tokens_seen": 488112128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303410230692076, + "loss": 2.7296, + "theoretical_loss": 3.92497808935481, + "tokens_seen": 488177664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 280726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.50590443611145, + "objective/train/theoretical_loss": 3.9249211063121283, + "objective/train/tokens_used": 508703200, + "theoretical_loss": 3.9249211063121283, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303309929789368, + "loss": 3.0183, + "theoretical_loss": 3.9249211063121283, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043032096288866603, + "loss": 2.6671, + "theoretical_loss": 3.9248641330589806, + "tokens_seen": 488308736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043031093279839516, + "loss": 2.8775, + "theoretical_loss": 3.924807169592372, + "tokens_seen": 488374272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303009027081244, + "loss": 2.9192, + "theoretical_loss": 3.9247502159093086, + "tokens_seen": 488439808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302908726178535, + "loss": 2.6838, + "theoretical_loss": 3.924693272006796, + "tokens_seen": 488505344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043028084252758276, + "loss": 2.9602, + "theoretical_loss": 3.924636337881845, + "tokens_seen": 488570880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043027081243731194, + "loss": 2.9434, + "theoretical_loss": 3.924579413531465, + "tokens_seen": 488636416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302607823470411, + "loss": 2.9598, + "theoretical_loss": 3.9245224989526664, + "tokens_seen": 488701952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302507522567703, + "loss": 2.862, + "theoretical_loss": 3.924465594142463, + "tokens_seen": 488767488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043024072216649953, + "loss": 2.9043, + "theoretical_loss": 3.9244086990978673, + "tokens_seen": 488833024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043023069207622866, + "loss": 2.8912, + "theoretical_loss": 3.924351813815895, + "tokens_seen": 488898560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302206619859579, + "loss": 2.9308, + "theoretical_loss": 3.9242949382935635, + "tokens_seen": 488964096 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430210631895687, + "loss": 2.9168, + "theoretical_loss": 3.9242380725278894, + "tokens_seen": 489029632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043020060180541626, + "loss": 2.723, + "theoretical_loss": 3.9241812165158922, + "tokens_seen": 489095168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043019057171514544, + "loss": 2.9341, + "theoretical_loss": 3.9241243702545923, + "tokens_seen": 489160704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301805416248746, + "loss": 2.7376, + "theoretical_loss": 3.9240675337410122, + "tokens_seen": 489226240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043017051153460386, + "loss": 2.9562, + "theoretical_loss": 3.924010706972174, + "tokens_seen": 489291776 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430160481444333, + "loss": 2.5616, + "theoretical_loss": 3.9239538899451025, + "tokens_seen": 489357312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301504513540622, + "loss": 2.9733, + "theoretical_loss": 3.923897082656823, + "tokens_seen": 489422848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301404212637914, + "loss": 2.7338, + "theoretical_loss": 3.9238402851043634, + "tokens_seen": 489488384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301303911735206, + "loss": 2.8406, + "theoretical_loss": 3.9237834972847505, + "tokens_seen": 489553920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043012036108324976, + "loss": 2.8607, + "theoretical_loss": 3.923726719195016, + "tokens_seen": 489619456 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430110330992979, + "loss": 2.7782, + "theoretical_loss": 3.9236699508321884, + "tokens_seen": 489684992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301003009027081, + "loss": 2.9235, + "theoretical_loss": 3.9236131921933017, + "tokens_seen": 489750528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043009027081243736, + "loss": 2.9848, + "theoretical_loss": 3.923556443275388, + "tokens_seen": 489816064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 281543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0093557834625244, + "objective/train/theoretical_loss": 3.923499704075483, + "objective/train/tokens_used": 510341600, + "theoretical_loss": 3.923499704075483, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300802407221665, + "loss": 2.8803, + "theoretical_loss": 3.923499704075483, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300702106318957, + "loss": 2.945, + "theoretical_loss": 3.923442974590623, + "tokens_seen": 489947136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300601805416249, + "loss": 2.7447, + "theoretical_loss": 3.923386254817845, + "tokens_seen": 490012672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300501504513541, + "loss": 3.1141, + "theoretical_loss": 3.9233295447541874, + "tokens_seen": 490078208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043004012036108326, + "loss": 2.761, + "theoretical_loss": 3.92327284439669, + "tokens_seen": 490143744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043003009027081245, + "loss": 2.8582, + "theoretical_loss": 3.9232161537423957, + "tokens_seen": 490209280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043002006018054163, + "loss": 2.824, + "theoretical_loss": 3.9231594727883454, + "tokens_seen": 490274816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043001003009027086, + "loss": 2.6088, + "theoretical_loss": 3.9231028015315834, + "tokens_seen": 490340352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043, + "loss": 2.857, + "theoretical_loss": 3.9230461399691556, + "tokens_seen": 490405888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299899699097292, + "loss": 2.7562, + "theoretical_loss": 3.922989488098107, + "tokens_seen": 490471424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042997993981945835, + "loss": 2.9854, + "theoretical_loss": 3.9229328459154873, + "tokens_seen": 490536960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299699097291876, + "loss": 2.8507, + "theoretical_loss": 3.9228762134183444, + "tokens_seen": 490602496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042995987963891677, + "loss": 3.0315, + "theoretical_loss": 3.922819590603729, + "tokens_seen": 490668032 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042994984954864595, + "loss": 2.8923, + "theoretical_loss": 3.922762977468692, + "tokens_seen": 490733568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042993981945837513, + "loss": 2.7771, + "theoretical_loss": 3.9227063740102874, + "tokens_seen": 490799104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042992978936810437, + "loss": 3.0867, + "theoretical_loss": 3.9226497802255684, + "tokens_seen": 490864640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299197592778335, + "loss": 2.6551, + "theoretical_loss": 3.922593196111592, + "tokens_seen": 490930176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042990972918756273, + "loss": 2.952, + "theoretical_loss": 3.9225366216654134, + "tokens_seen": 490995712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042989969909729185, + "loss": 2.7502, + "theoretical_loss": 3.9224800568840914, + "tokens_seen": 491061248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298896690070211, + "loss": 2.7318, + "theoretical_loss": 3.922423501764686, + "tokens_seen": 491126784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042987963891675027, + "loss": 3.0758, + "theoretical_loss": 3.9223669563042565, + "tokens_seen": 491192320 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042986960882647945, + "loss": 2.9801, + "theoretical_loss": 3.9223104204998664, + "tokens_seen": 491257856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042985957873620863, + "loss": 2.9669, + "theoretical_loss": 3.922253894348578, + "tokens_seen": 491323392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298495486459378, + "loss": 2.9609, + "theoretical_loss": 3.922197377847456, + "tokens_seen": 491388928 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429839518555667, + "loss": 2.9549, + "theoretical_loss": 3.922140870993566, + "tokens_seen": 491454464 + }, + { + "debugging/Self-BLEU-5": 0.364302224771738, + "debugging/distinct-1-grams": 0.7813905547226386, + "debugging/distinct-2-grams": 0.9746492802208804, + "debugging/entropy-1-grams": 5.062092644878832, + "debugging/entropy-2-grams": 5.611156931113771, + "debugging/length": 539.0, + "debugging/num_segments": 4, + "epoch": 0.15, + "objective/train/docs_used": 282054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8211379051208496, + "objective/train/theoretical_loss": 3.9220843737839752, + "objective/train/tokens_used": 511980000, + "theoretical_loss": 3.9220843737839752, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042982948846539623, + "loss": 2.85, + "theoretical_loss": 3.9220843737839752, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042981945837512536, + "loss": 2.6656, + "theoretical_loss": 3.9220278862157523, + "tokens_seen": 491585536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298094282848546, + "loss": 2.8264, + "theoretical_loss": 3.921971408285967, + "tokens_seen": 491651072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297993981945837, + "loss": 2.8791, + "theoretical_loss": 3.92191493999169, + "tokens_seen": 491716608 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042978936810431296, + "loss": 2.8162, + "theoretical_loss": 3.921858481329994, + "tokens_seen": 491782144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042977933801404214, + "loss": 3.1575, + "theoretical_loss": 3.9218020322979514, + "tokens_seen": 491847680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297693079237713, + "loss": 2.671, + "theoretical_loss": 3.9217455928926377, + "tokens_seen": 491913216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297592778335005, + "loss": 2.9984, + "theoretical_loss": 3.9216891631111293, + "tokens_seen": 491978752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042974924774322973, + "loss": 2.8436, + "theoretical_loss": 3.9216327429505027, + "tokens_seen": 492044288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042973921765295886, + "loss": 2.7374, + "theoretical_loss": 3.921576332407837, + "tokens_seen": 492109824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297291875626881, + "loss": 3.0636, + "theoretical_loss": 3.921519931480213, + "tokens_seen": 492175360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297191574724172, + "loss": 3.0291, + "theoretical_loss": 3.92146354016471, + "tokens_seen": 492240896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042970912738214646, + "loss": 2.9755, + "theoretical_loss": 3.921407158458411, + "tokens_seen": 492306432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042969909729187564, + "loss": 2.7441, + "theoretical_loss": 3.9213507863584014, + "tokens_seen": 492371968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296890672016048, + "loss": 2.8323, + "theoretical_loss": 3.921294423861764, + "tokens_seen": 492437504 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429679037111334, + "loss": 2.925, + "theoretical_loss": 3.921238070965586, + "tokens_seen": 492503040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296690070210632, + "loss": 2.81, + "theoretical_loss": 3.921181727666955, + "tokens_seen": 492568576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042965897693079236, + "loss": 2.8812, + "theoretical_loss": 3.92112539396296, + "tokens_seen": 492634112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296489468405216, + "loss": 3.0883, + "theoretical_loss": 3.9210690698506907, + "tokens_seen": 492699648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296389167502507, + "loss": 3.0465, + "theoretical_loss": 3.9210127553272383, + "tokens_seen": 492765184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042962888665997996, + "loss": 2.9085, + "theoretical_loss": 3.9209564503896956, + "tokens_seen": 492830720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296188565697091, + "loss": 2.6728, + "theoretical_loss": 3.9209001550351563, + "tokens_seen": 492896256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296088264794383, + "loss": 2.7792, + "theoretical_loss": 3.9208438692607164, + "tokens_seen": 492961792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295987963891675, + "loss": 2.9342, + "theoretical_loss": 3.920787593063471, + "tokens_seen": 493027328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295887662988967, + "loss": 3.0404, + "theoretical_loss": 3.9207313264405186, + "tokens_seen": 493092864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 282706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2857396602630615, + "objective/train/theoretical_loss": 3.9206750693889583, + "objective/train/tokens_used": 513618400, + "theoretical_loss": 3.9206750693889583, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042957873620862587, + "loss": 2.9551, + "theoretical_loss": 3.9206750693889583, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295687061183551, + "loss": 2.7466, + "theoretical_loss": 3.9206188219058897, + "tokens_seen": 493223936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042955867602808423, + "loss": 2.7163, + "theoretical_loss": 3.9205625839884144, + "tokens_seen": 493289472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042954864593781347, + "loss": 2.7523, + "theoretical_loss": 3.920506355633636, + "tokens_seen": 493355008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295386158475426, + "loss": 2.9156, + "theoretical_loss": 3.9204501368386575, + "tokens_seen": 493420544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042952858575727183, + "loss": 2.8954, + "theoretical_loss": 3.9203939276005846, + "tokens_seen": 493486080 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429518555667001, + "loss": 2.885, + "theoretical_loss": 3.9203377279165235, + "tokens_seen": 493551616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295085255767302, + "loss": 2.812, + "theoretical_loss": 3.920281537783582, + "tokens_seen": 493617152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042949849548645937, + "loss": 2.797, + "theoretical_loss": 3.9202253571988694, + "tokens_seen": 493682688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042948846539618855, + "loss": 2.858, + "theoretical_loss": 3.9201691861594963, + "tokens_seen": 493748224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042947843530591773, + "loss": 2.8976, + "theoretical_loss": 3.9201130246625735, + "tokens_seen": 493813760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042946840521564697, + "loss": 2.9483, + "theoretical_loss": 3.920056872705214, + "tokens_seen": 493879296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294583751253761, + "loss": 2.8325, + "theoretical_loss": 3.9200007302845323, + "tokens_seen": 493944832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042944834503510533, + "loss": 2.9839, + "theoretical_loss": 3.9199445973976434, + "tokens_seen": 494010368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294383149448345, + "loss": 2.9204, + "theoretical_loss": 3.9198884740416644, + "tokens_seen": 494075904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294282848545637, + "loss": 2.8143, + "theoretical_loss": 3.9198323602137126, + "tokens_seen": 494141440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042941825476429293, + "loss": 3.0728, + "theoretical_loss": 3.9197762559109073, + "tokens_seen": 494206976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042940822467402206, + "loss": 2.9142, + "theoretical_loss": 3.919720161130368, + "tokens_seen": 494272512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293981945837513, + "loss": 3.0248, + "theoretical_loss": 3.9196640758692176, + "tokens_seen": 494338048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042938816449348047, + "loss": 2.6867, + "theoretical_loss": 3.9196080001245783, + "tokens_seen": 494403584 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042937813440320965, + "loss": 2.8787, + "theoretical_loss": 3.9195519338935743, + "tokens_seen": 494469120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042936810431293883, + "loss": 2.6973, + "theoretical_loss": 3.919495877173331, + "tokens_seen": 494534656 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429358074222668, + "loss": 2.8302, + "theoretical_loss": 3.9194398299609743, + "tokens_seen": 494600192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293480441323972, + "loss": 3.1014, + "theoretical_loss": 3.919383792253633, + "tokens_seen": 494665728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042933801404212643, + "loss": 2.8187, + "theoretical_loss": 3.919327764048437, + "tokens_seen": 494731264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 283457, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3519198894500732, + "objective/train/theoretical_loss": 3.919271745342514, + "objective/train/tokens_used": 515256800, + "theoretical_loss": 3.919271745342514, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042932798395185556, + "loss": 2.9657, + "theoretical_loss": 3.919271745342514, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293179538615848, + "loss": 2.8827, + "theoretical_loss": 3.9192157361329976, + "tokens_seen": 494862336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293079237713139, + "loss": 2.8081, + "theoretical_loss": 3.9191597364170208, + "tokens_seen": 494927872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042929789368104316, + "loss": 2.9318, + "theoretical_loss": 3.919103746191716, + "tokens_seen": 494993408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042928786359077234, + "loss": 2.7932, + "theoretical_loss": 3.91904776545422, + "tokens_seen": 495058944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292778335005015, + "loss": 2.8108, + "theoretical_loss": 3.918991794201669, + "tokens_seen": 495124480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292678034102307, + "loss": 2.684, + "theoretical_loss": 3.9189358324312007, + "tokens_seen": 495190016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042925777331995993, + "loss": 2.6035, + "theoretical_loss": 3.9188798801399543, + "tokens_seen": 495255552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042924774322968906, + "loss": 2.8571, + "theoretical_loss": 3.9188239373250697, + "tokens_seen": 495321088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292377131394183, + "loss": 2.8791, + "theoretical_loss": 3.9187680039836885, + "tokens_seen": 495386624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292276830491474, + "loss": 2.9489, + "theoretical_loss": 3.918712080112954, + "tokens_seen": 495452160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042921765295887666, + "loss": 2.83, + "theoretical_loss": 3.9186561657100096, + "tokens_seen": 495517696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042920762286860584, + "loss": 2.736, + "theoretical_loss": 3.918600260772001, + "tokens_seen": 495583232 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429197592778335, + "loss": 2.7423, + "theoretical_loss": 3.9185443652960745, + "tokens_seen": 495648768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291875626880642, + "loss": 3.0763, + "theoretical_loss": 3.918488479279378, + "tokens_seen": 495714304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291775325977934, + "loss": 2.8709, + "theoretical_loss": 3.9184326027190606, + "tokens_seen": 495779840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042916750250752256, + "loss": 2.9463, + "theoretical_loss": 3.9183767356122714, + "tokens_seen": 495845376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291574724172518, + "loss": 3.1239, + "theoretical_loss": 3.9183208779561634, + "tokens_seen": 495910912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291474423269809, + "loss": 2.9989, + "theoretical_loss": 3.9182650297478885, + "tokens_seen": 495976448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042913741223671016, + "loss": 2.8044, + "theoretical_loss": 3.9182091909846, + "tokens_seen": 496041984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291273821464393, + "loss": 2.9178, + "theoretical_loss": 3.9181533616634545, + "tokens_seen": 496107520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291173520561685, + "loss": 3.0983, + "theoretical_loss": 3.918097541781607, + "tokens_seen": 496173056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291073219658977, + "loss": 3.0364, + "theoretical_loss": 3.9180417313362157, + "tokens_seen": 496238592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290972918756269, + "loss": 2.9606, + "theoretical_loss": 3.9179859303244395, + "tokens_seen": 496304128 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042908726178535607, + "loss": 2.6999, + "theoretical_loss": 3.917930138743438, + "tokens_seen": 496369664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 284572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4066948890686035, + "objective/train/theoretical_loss": 3.9178743565903735, + "objective/train/tokens_used": 516895200, + "theoretical_loss": 3.9178743565903735, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290772316950853, + "loss": 2.7991, + "theoretical_loss": 3.9178743565903735, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042906720160481443, + "loss": 2.9311, + "theoretical_loss": 3.9178185838624078, + "tokens_seen": 496500736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042905717151454367, + "loss": 2.6928, + "theoretical_loss": 3.917762820556705, + "tokens_seen": 496566272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290471414242728, + "loss": 2.8094, + "theoretical_loss": 3.9177070666704292, + "tokens_seen": 496631808 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042903711133400203, + "loss": 2.7391, + "theoretical_loss": 3.917651322200747, + "tokens_seen": 496697344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290270812437312, + "loss": 2.8844, + "theoretical_loss": 3.9175955871448265, + "tokens_seen": 496762880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290170511534604, + "loss": 2.9062, + "theoretical_loss": 3.917539861499836, + "tokens_seen": 496828416 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042900702106318957, + "loss": 2.9001, + "theoretical_loss": 3.9174841452629456, + "tokens_seen": 496893952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042899699097291875, + "loss": 3.0454, + "theoretical_loss": 3.9174284384313256, + "tokens_seen": 496959488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042898696088264793, + "loss": 2.9205, + "theoretical_loss": 3.917372741002149, + "tokens_seen": 497025024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042897693079237717, + "loss": 2.9715, + "theoretical_loss": 3.917317052972589, + "tokens_seen": 497090560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289669007021063, + "loss": 2.823, + "theoretical_loss": 3.9172613743398204, + "tokens_seen": 497156096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042895687061183553, + "loss": 2.7293, + "theoretical_loss": 3.91720570510102, + "tokens_seen": 497221632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289468405215647, + "loss": 2.8304, + "theoretical_loss": 3.917150045253364, + "tokens_seen": 497287168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289368104312939, + "loss": 2.9458, + "theoretical_loss": 3.917094394794031, + "tokens_seen": 497352704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289267803410231, + "loss": 3.1032, + "theoretical_loss": 3.917038753720201, + "tokens_seen": 497418240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042891675025075226, + "loss": 2.8554, + "theoretical_loss": 3.9169831220290554, + "tokens_seen": 497483776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042890672016048144, + "loss": 2.9893, + "theoretical_loss": 3.916927499717775, + "tokens_seen": 497549312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042889669007021067, + "loss": 2.8381, + "theoretical_loss": 3.916871886783544, + "tokens_seen": 497614848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288866599799398, + "loss": 2.9519, + "theoretical_loss": 3.9168162832235467, + "tokens_seen": 497680384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042887662988966903, + "loss": 3.1386, + "theoretical_loss": 3.9167606890349687, + "tokens_seen": 497745920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042886659979939816, + "loss": 2.5596, + "theoretical_loss": 3.9167051042149974, + "tokens_seen": 497811456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288565697091274, + "loss": 2.9973, + "theoretical_loss": 3.9166495287608205, + "tokens_seen": 497876992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288465396188566, + "loss": 2.8093, + "theoretical_loss": 3.9165939626696273, + "tokens_seen": 497942528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042883650952858576, + "loss": 2.8571, + "theoretical_loss": 3.916538405938609, + "tokens_seen": 498008064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 285190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.950747489929199, + "objective/train/theoretical_loss": 3.916482858564957, + "objective/train/tokens_used": 518533600, + "theoretical_loss": 3.916482858564957, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042882647943831494, + "loss": 2.9687, + "theoretical_loss": 3.916482858564957, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288164493480441, + "loss": 2.7534, + "theoretical_loss": 3.916427320545864, + "tokens_seen": 498139136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288064192577733, + "loss": 2.8904, + "theoretical_loss": 3.9163717918785252, + "tokens_seen": 498204672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042879638916750254, + "loss": 2.9187, + "theoretical_loss": 3.9163162725601355, + "tokens_seen": 498270208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878635907723166, + "loss": 2.6975, + "theoretical_loss": 3.916260762587892, + "tokens_seen": 498335744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287763289869609, + "loss": 2.695, + "theoretical_loss": 3.916205261958991, + "tokens_seen": 498401280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287662988966901, + "loss": 2.7584, + "theoretical_loss": 3.916149770670633, + "tokens_seen": 498466816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042875626880641926, + "loss": 2.8199, + "theoretical_loss": 3.9160942887200183, + "tokens_seen": 498532352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042874623871614844, + "loss": 2.6627, + "theoretical_loss": 3.916038816104348, + "tokens_seen": 498597888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287362086258776, + "loss": 2.9793, + "theoretical_loss": 3.9159833528208248, + "tokens_seen": 498663424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287261785356068, + "loss": 2.6124, + "theoretical_loss": 3.915927898866653, + "tokens_seen": 498728960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042871614844533604, + "loss": 2.8027, + "theoretical_loss": 3.915872454239037, + "tokens_seen": 498794496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042870611835506517, + "loss": 3.0653, + "theoretical_loss": 3.9158170189351833, + "tokens_seen": 498860032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286960882647944, + "loss": 3.1196, + "theoretical_loss": 3.9157615929523004, + "tokens_seen": 498925568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042868605817452353, + "loss": 2.8093, + "theoretical_loss": 3.9157061762875953, + "tokens_seen": 498991104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042867602808425276, + "loss": 2.8654, + "theoretical_loss": 3.9156507689382796, + "tokens_seen": 499056640 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428665997993982, + "loss": 2.7376, + "theoretical_loss": 3.915595370901563, + "tokens_seen": 499122176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286559679037111, + "loss": 2.7512, + "theoretical_loss": 3.915539982174659, + "tokens_seen": 499187712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042864593781344036, + "loss": 3.015, + "theoretical_loss": 3.9154846027547805, + "tokens_seen": 499253248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286359077231695, + "loss": 2.6258, + "theoretical_loss": 3.915429232639142, + "tokens_seen": 499318784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286258776328987, + "loss": 2.9637, + "theoretical_loss": 3.9153738718249604, + "tokens_seen": 499384320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286158475426279, + "loss": 2.8947, + "theoretical_loss": 3.9153185203094516, + "tokens_seen": 499449856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286058174523571, + "loss": 3.0437, + "theoretical_loss": 3.915263178089835, + "tokens_seen": 499515392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042859578736208627, + "loss": 3.0086, + "theoretical_loss": 3.915207845163329, + "tokens_seen": 499580928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285857572718155, + "loss": 2.7415, + "theoretical_loss": 3.915152521527155, + "tokens_seen": 499646464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 286367, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.678431272506714, + "objective/train/theoretical_loss": 3.915097207178535, + "objective/train/tokens_used": 520172000, + "theoretical_loss": 3.915097207178535, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042857572718154463, + "loss": 2.8241, + "theoretical_loss": 3.915097207178535, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042856569709127387, + "loss": 2.8393, + "theoretical_loss": 3.9150419021146927, + "tokens_seen": 499777536 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428555667001003, + "loss": 2.9709, + "theoretical_loss": 3.9149866063328504, + "tokens_seen": 499843072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854563691073223, + "loss": 2.9225, + "theoretical_loss": 3.914931319830236, + "tokens_seen": 499908608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285356068204614, + "loss": 2.8198, + "theoretical_loss": 3.9148760426040736, + "tokens_seen": 499974144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285255767301906, + "loss": 2.7686, + "theoretical_loss": 3.9148207746515933, + "tokens_seen": 500039680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042851554663991977, + "loss": 2.7698, + "theoretical_loss": 3.9147655159700236, + "tokens_seen": 500105216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042850551654964895, + "loss": 2.8607, + "theoretical_loss": 3.914710266556594, + "tokens_seen": 500170752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042849548645937813, + "loss": 2.9713, + "theoretical_loss": 3.9146550264085374, + "tokens_seen": 500236288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042848545636910737, + "loss": 2.6836, + "theoretical_loss": 3.9145997955230842, + "tokens_seen": 500301824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284754262788365, + "loss": 2.6966, + "theoretical_loss": 3.9145445738974702, + "tokens_seen": 500367360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846539618856573, + "loss": 2.6121, + "theoretical_loss": 3.91448936152893, + "tokens_seen": 500432896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284553660982949, + "loss": 2.7794, + "theoretical_loss": 3.914434158414699, + "tokens_seen": 500498432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284453360080241, + "loss": 3.1444, + "theoretical_loss": 3.9143789645520153, + "tokens_seen": 500563968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284353059177533, + "loss": 2.7484, + "theoretical_loss": 3.914323779938117, + "tokens_seen": 500629504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042842527582748246, + "loss": 2.6557, + "theoretical_loss": 3.9142686045702444, + "tokens_seen": 500695040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042841524573721164, + "loss": 2.847, + "theoretical_loss": 3.914213438445638, + "tokens_seen": 500760576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042840521564694087, + "loss": 3.133, + "theoretical_loss": 3.9141582815615408, + "tokens_seen": 500826112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042839518555667, + "loss": 2.9062, + "theoretical_loss": 3.9141031339151953, + "tokens_seen": 500891648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042838515546639923, + "loss": 2.8711, + "theoretical_loss": 3.9140479955038456, + "tokens_seen": 500957184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042837512537612836, + "loss": 2.7578, + "theoretical_loss": 3.9139928663247385, + "tokens_seen": 501022720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283650952858576, + "loss": 2.7539, + "theoretical_loss": 3.9139377463751197, + "tokens_seen": 501088256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283550651955868, + "loss": 2.7564, + "theoretical_loss": 3.913882635652238, + "tokens_seen": 501153792 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042834503510531596, + "loss": 2.7827, + "theoretical_loss": 3.9138275341533424, + "tokens_seen": 501219328 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042833500501504514, + "loss": 2.7285, + "theoretical_loss": 3.913772441875684, + "tokens_seen": 501284864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 286935, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.952582359313965, + "objective/train/theoretical_loss": 3.9137173588165135, + "objective/train/tokens_used": 521810400, + "theoretical_loss": 3.9137173588165135, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283249749247743, + "loss": 2.888, + "theoretical_loss": 3.9137173588165135, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283149448345035, + "loss": 2.7366, + "theoretical_loss": 3.9136622849730833, + "tokens_seen": 501415936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042830491474423274, + "loss": 2.6463, + "theoretical_loss": 3.913607220342648, + "tokens_seen": 501481472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042829488465396186, + "loss": 2.785, + "theoretical_loss": 3.9135521649224634, + "tokens_seen": 501547008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282848545636911, + "loss": 2.9083, + "theoretical_loss": 3.9134971187097847, + "tokens_seen": 501612544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282748244734203, + "loss": 2.886, + "theoretical_loss": 3.913442081701869, + "tokens_seen": 501678080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042826479438314946, + "loss": 2.6593, + "theoretical_loss": 3.9133870538959763, + "tokens_seen": 501743616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042825476429287864, + "loss": 2.8406, + "theoretical_loss": 3.9133320352893657, + "tokens_seen": 501809152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282447342026078, + "loss": 2.881, + "theoretical_loss": 3.913277025879298, + "tokens_seen": 501874688 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428234704112337, + "loss": 2.8322, + "theoretical_loss": 3.9132220256630363, + "tokens_seen": 501940224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042822467402206624, + "loss": 2.919, + "theoretical_loss": 3.9131670346378424, + "tokens_seen": 502005760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042821464393179537, + "loss": 2.8986, + "theoretical_loss": 3.913112052800982, + "tokens_seen": 502071296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282046138415246, + "loss": 3.0142, + "theoretical_loss": 3.9130570801497204, + "tokens_seen": 502136832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042819458375125373, + "loss": 2.8962, + "theoretical_loss": 3.9130021166813247, + "tokens_seen": 502202368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042818455366098296, + "loss": 2.9193, + "theoretical_loss": 3.9129471623930625, + "tokens_seen": 502267904 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042817452357071215, + "loss": 2.6178, + "theoretical_loss": 3.912892217282203, + "tokens_seen": 502333440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281644934804413, + "loss": 2.7213, + "theoretical_loss": 3.912837281346017, + "tokens_seen": 502398976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281544633901705, + "loss": 3.0234, + "theoretical_loss": 3.9127823545817755, + "tokens_seen": 502464512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281444332998997, + "loss": 2.8943, + "theoretical_loss": 3.9127274369867515, + "tokens_seen": 502530048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042813440320962887, + "loss": 3.3673, + "theoretical_loss": 3.912672528558219, + "tokens_seen": 502595584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281243731193581, + "loss": 2.9674, + "theoretical_loss": 3.9126176292934525, + "tokens_seen": 502661120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042811434302908723, + "loss": 2.7487, + "theoretical_loss": 3.9125627391897284, + "tokens_seen": 502726656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042810431293881647, + "loss": 2.9542, + "theoretical_loss": 3.9125078582443247, + "tokens_seen": 502792192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042809428284854565, + "loss": 2.7127, + "theoretical_loss": 3.912452986454519, + "tokens_seen": 502857728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042808425275827483, + "loss": 2.9038, + "theoretical_loss": 3.9123981238175913, + "tokens_seen": 502923264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 288241, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.815506935119629, + "objective/train/theoretical_loss": 3.9123432703308225, + "objective/train/tokens_used": 523448800, + "theoretical_loss": 3.9123432703308225, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428074222668004, + "loss": 2.8686, + "theoretical_loss": 3.9123432703308225, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280641925777332, + "loss": 2.8739, + "theoretical_loss": 3.9122884259914947, + "tokens_seen": 503054336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280541624874624, + "loss": 2.9241, + "theoretical_loss": 3.912233590796891, + "tokens_seen": 503119872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280441323971916, + "loss": 2.8723, + "theoretical_loss": 3.912178764744296, + "tokens_seen": 503185408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042803410230692074, + "loss": 2.7677, + "theoretical_loss": 3.9121239478309944, + "tokens_seen": 503250944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802407221664997, + "loss": 2.8224, + "theoretical_loss": 3.9120691400542738, + "tokens_seen": 503316480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280140421263791, + "loss": 2.6969, + "theoretical_loss": 3.912014341411421, + "tokens_seen": 503382016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042800401203610833, + "loss": 2.9163, + "theoretical_loss": 3.9119595518997254, + "tokens_seen": 503447552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279939819458375, + "loss": 3.1166, + "theoretical_loss": 3.911904771516477, + "tokens_seen": 503513088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279839518555667, + "loss": 2.8336, + "theoretical_loss": 3.9118500002589682, + "tokens_seen": 503578624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279739217652959, + "loss": 2.8036, + "theoretical_loss": 3.9117952381244896, + "tokens_seen": 503644160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279638916750251, + "loss": 2.8712, + "theoretical_loss": 3.911740485110337, + "tokens_seen": 503709696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042795386158475424, + "loss": 2.8051, + "theoretical_loss": 3.9116857412138026, + "tokens_seen": 503775232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279438314944835, + "loss": 2.9153, + "theoretical_loss": 3.9116310064321835, + "tokens_seen": 503840768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279338014042126, + "loss": 2.7278, + "theoretical_loss": 3.9115762807627776, + "tokens_seen": 503906304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042792377131394184, + "loss": 2.814, + "theoretical_loss": 3.9115215642028813, + "tokens_seen": 503971840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042791374122367107, + "loss": 2.7999, + "theoretical_loss": 3.911466856749796, + "tokens_seen": 504037376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279037111334002, + "loss": 2.8065, + "theoretical_loss": 3.911412158400821, + "tokens_seen": 504102912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042789368104312943, + "loss": 2.8346, + "theoretical_loss": 3.9113574691532573, + "tokens_seen": 504168448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042788365095285856, + "loss": 3.0613, + "theoretical_loss": 3.911302789004409, + "tokens_seen": 504233984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278736208625878, + "loss": 2.9722, + "theoretical_loss": 3.9112481179515797, + "tokens_seen": 504299520 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427863590772317, + "loss": 2.8922, + "theoretical_loss": 3.9111934559920742, + "tokens_seen": 504365056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042785356068204616, + "loss": 2.7941, + "theoretical_loss": 3.9111388031231993, + "tokens_seen": 504430592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042784353059177534, + "loss": 2.9636, + "theoretical_loss": 3.911084159342261, + "tokens_seen": 504496128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278335005015045, + "loss": 2.7491, + "theoretical_loss": 3.91102952464657, + "tokens_seen": 504561664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 289179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2568519115448, + "objective/train/theoretical_loss": 3.9109748990334348, + "objective/train/tokens_used": 525087200, + "theoretical_loss": 3.9109748990334348, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278234704112337, + "loss": 2.8859, + "theoretical_loss": 3.9109748990334348, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042781344032096294, + "loss": 2.7951, + "theoretical_loss": 3.910920282500166, + "tokens_seen": 504692736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042780341023069206, + "loss": 3.0004, + "theoretical_loss": 3.910865675044076, + "tokens_seen": 504758272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277933801404213, + "loss": 2.8993, + "theoretical_loss": 3.9108110766624775, + "tokens_seen": 504823808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277833500501505, + "loss": 2.6838, + "theoretical_loss": 3.9107564873526854, + "tokens_seen": 504889344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042777331995987966, + "loss": 2.6696, + "theoretical_loss": 3.9107019071120153, + "tokens_seen": 504954880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042776328986960884, + "loss": 2.7526, + "theoretical_loss": 3.910647335937783, + "tokens_seen": 505020416 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427753259779338, + "loss": 3.007, + "theoretical_loss": 3.910592773827306, + "tokens_seen": 505085952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277432296890672, + "loss": 2.6543, + "theoretical_loss": 3.9105382207779043, + "tokens_seen": 505151488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042773319959879644, + "loss": 2.9892, + "theoretical_loss": 3.9104836767868973, + "tokens_seen": 505217024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042772316950852557, + "loss": 2.8852, + "theoretical_loss": 3.9104291418516057, + "tokens_seen": 505282560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277131394182548, + "loss": 2.5701, + "theoretical_loss": 3.910374615969352, + "tokens_seen": 505348096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042770310932798393, + "loss": 3.0667, + "theoretical_loss": 3.9103200991374605, + "tokens_seen": 505413632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042769307923771316, + "loss": 2.8631, + "theoretical_loss": 3.9102655913532542, + "tokens_seen": 505479168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042768304914744235, + "loss": 2.7267, + "theoretical_loss": 3.9102110926140603, + "tokens_seen": 505544704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042767301905717153, + "loss": 2.6952, + "theoretical_loss": 3.910156602917205, + "tokens_seen": 505610240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276629889669007, + "loss": 2.6144, + "theoretical_loss": 3.9101021222600156, + "tokens_seen": 505675776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276529588766299, + "loss": 2.9215, + "theoretical_loss": 3.9100476506398216, + "tokens_seen": 505741312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042764292878635907, + "loss": 3.0363, + "theoretical_loss": 3.909993188053954, + "tokens_seen": 505806848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276328986960883, + "loss": 2.9977, + "theoretical_loss": 3.909938734499743, + "tokens_seen": 505872384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042762286860581743, + "loss": 2.7794, + "theoretical_loss": 3.9098842899745216, + "tokens_seen": 505937920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042761283851554667, + "loss": 2.673, + "theoretical_loss": 3.909829854475624, + "tokens_seen": 506003456 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042760280842527585, + "loss": 2.8276, + "theoretical_loss": 3.9097754280003842, + "tokens_seen": 506068992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042759277833500503, + "loss": 2.754, + "theoretical_loss": 3.9097210105461384, + "tokens_seen": 506134528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275827482447342, + "loss": 3.0903, + "theoretical_loss": 3.9096666021102235, + "tokens_seen": 506200064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 289852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1201510429382324, + "objective/train/theoretical_loss": 3.909612202689977, + "objective/train/tokens_used": 526725600, + "theoretical_loss": 3.909612202689977, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275727181544634, + "loss": 2.8416, + "theoretical_loss": 3.909612202689977, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275626880641926, + "loss": 2.6884, + "theoretical_loss": 3.90955781228274, + "tokens_seen": 506331136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275526579739218, + "loss": 2.7749, + "theoretical_loss": 3.909503430885851, + "tokens_seen": 506396672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042754262788365094, + "loss": 2.8646, + "theoretical_loss": 3.9094490584966524, + "tokens_seen": 506462208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042753259779338017, + "loss": 2.8293, + "theoretical_loss": 3.9093946951124874, + "tokens_seen": 506527744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275225677031093, + "loss": 2.9254, + "theoretical_loss": 3.9093403407306986, + "tokens_seen": 506593280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042751253761283853, + "loss": 2.8896, + "theoretical_loss": 3.9092859953486316, + "tokens_seen": 506658816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275025075225677, + "loss": 2.9322, + "theoretical_loss": 3.909231658963633, + "tokens_seen": 506724352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274924774322969, + "loss": 2.8241, + "theoretical_loss": 3.9091773315730487, + "tokens_seen": 506789888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274824473420261, + "loss": 2.8595, + "theoretical_loss": 3.909123013174228, + "tokens_seen": 506855424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274724172517553, + "loss": 3.029, + "theoretical_loss": 3.90906870376452, + "tokens_seen": 506920960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042746238716148444, + "loss": 2.889, + "theoretical_loss": 3.909014403341275, + "tokens_seen": 506986496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274523570712137, + "loss": 3.0323, + "theoretical_loss": 3.9089601119018456, + "tokens_seen": 507052032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274423269809428, + "loss": 3.1528, + "theoretical_loss": 3.908905829443583, + "tokens_seen": 507117568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042743229689067204, + "loss": 2.9999, + "theoretical_loss": 3.9088515559638424, + "tokens_seen": 507183104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274222668004012, + "loss": 2.8889, + "theoretical_loss": 3.9087972914599787, + "tokens_seen": 507248640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274122367101304, + "loss": 2.9263, + "theoretical_loss": 3.908743035929348, + "tokens_seen": 507314176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274022066198596, + "loss": 2.697, + "theoretical_loss": 3.9086887893693065, + "tokens_seen": 507379712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042739217652958876, + "loss": 2.9639, + "theoretical_loss": 3.9086345517772143, + "tokens_seen": 507445248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042738214643931794, + "loss": 2.8796, + "theoretical_loss": 3.90858032315043, + "tokens_seen": 507510784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273721163490472, + "loss": 2.7631, + "theoretical_loss": 3.9085261034863144, + "tokens_seen": 507576320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273620862587763, + "loss": 3.0314, + "theoretical_loss": 3.908471892782229, + "tokens_seen": 507641856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042735205616850554, + "loss": 2.9613, + "theoretical_loss": 3.9084176910355373, + "tokens_seen": 507707392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042734202607823467, + "loss": 3.1484, + "theoretical_loss": 3.9083634982436033, + "tokens_seen": 507772928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273319959879639, + "loss": 2.9547, + "theoretical_loss": 3.908309314403791, + "tokens_seen": 507838464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 291045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8749454021453857, + "objective/train/theoretical_loss": 3.908255139513467, + "objective/train/tokens_used": 528364000, + "theoretical_loss": 3.908255139513467, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273219658976931, + "loss": 3.0369, + "theoretical_loss": 3.908255139513467, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042731193580742226, + "loss": 2.8929, + "theoretical_loss": 3.9082009735699996, + "tokens_seen": 507969536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042730190571715145, + "loss": 2.7172, + "theoretical_loss": 3.9081468165707562, + "tokens_seen": 508035072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272918756268807, + "loss": 2.8316, + "theoretical_loss": 3.9080926685131074, + "tokens_seen": 508100608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272818455366098, + "loss": 2.7872, + "theoretical_loss": 3.908038529394423, + "tokens_seen": 508166144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042727181544633904, + "loss": 2.8132, + "theoretical_loss": 3.907984399212075, + "tokens_seen": 508231680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042726178535606817, + "loss": 2.9742, + "theoretical_loss": 3.9079302779634366, + "tokens_seen": 508297216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272517552657974, + "loss": 2.7523, + "theoretical_loss": 3.907876165645881, + "tokens_seen": 508362752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272417251755266, + "loss": 2.9132, + "theoretical_loss": 3.907822062256784, + "tokens_seen": 508428288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042723169508525577, + "loss": 2.6844, + "theoretical_loss": 3.9077679677935224, + "tokens_seen": 508493824 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042722166499498495, + "loss": 2.5976, + "theoretical_loss": 3.9077138822534723, + "tokens_seen": 508559360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042721163490471413, + "loss": 2.9316, + "theoretical_loss": 3.9076598056340126, + "tokens_seen": 508624896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272016048144433, + "loss": 3.027, + "theoretical_loss": 3.9076057379325233, + "tokens_seen": 508690432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042719157472417255, + "loss": 2.8114, + "theoretical_loss": 3.9075516791463842, + "tokens_seen": 508755968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271815446339017, + "loss": 2.8752, + "theoretical_loss": 3.907497629272978, + "tokens_seen": 508821504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271715145436309, + "loss": 2.7832, + "theoretical_loss": 3.907443588309687, + "tokens_seen": 508887040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271614844533601, + "loss": 2.9496, + "theoretical_loss": 3.9073895562538956, + "tokens_seen": 508952576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042715145436308927, + "loss": 2.7524, + "theoretical_loss": 3.907335533102988, + "tokens_seen": 509018112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271414242728185, + "loss": 2.7836, + "theoretical_loss": 3.9072815188543517, + "tokens_seen": 509083648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042713139418254763, + "loss": 2.78, + "theoretical_loss": 3.907227513505373, + "tokens_seen": 509149184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042712136409227687, + "loss": 3.1176, + "theoretical_loss": 3.9071735170534407, + "tokens_seen": 509214720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042711133400200605, + "loss": 2.789, + "theoretical_loss": 3.9071195294959438, + "tokens_seen": 509280256 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042710130391173523, + "loss": 2.7216, + "theoretical_loss": 3.9070655508302736, + "tokens_seen": 509345792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270912738214644, + "loss": 2.9398, + "theoretical_loss": 3.9070115810538217, + "tokens_seen": 509411328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270812437311936, + "loss": 2.7936, + "theoretical_loss": 3.9069576201639804, + "tokens_seen": 509476864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 291444, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.291428804397583, + "objective/train/theoretical_loss": 3.9069036681581446, + "objective/train/tokens_used": 530002400, + "theoretical_loss": 3.9069036681581446, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270712136409228, + "loss": 2.9279, + "theoretical_loss": 3.9069036681581446, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427061183550652, + "loss": 3.0056, + "theoretical_loss": 3.906849725033708, + "tokens_seen": 509607936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042705115346038114, + "loss": 2.9895, + "theoretical_loss": 3.906795790788067, + "tokens_seen": 509673472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042704112337011037, + "loss": 2.8832, + "theoretical_loss": 3.9067418654186197, + "tokens_seen": 509739008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270310932798395, + "loss": 2.8935, + "theoretical_loss": 3.9066879489227633, + "tokens_seen": 509804544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042702106318956873, + "loss": 2.7562, + "theoretical_loss": 3.906634041297898, + "tokens_seen": 509870080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270110330992979, + "loss": 3.0871, + "theoretical_loss": 3.906580142541424, + "tokens_seen": 509935616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270010030090271, + "loss": 2.7015, + "theoretical_loss": 3.906526252650743, + "tokens_seen": 510001152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269909729187563, + "loss": 3.0532, + "theoretical_loss": 3.9064723716232574, + "tokens_seen": 510066688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269809428284855, + "loss": 2.7929, + "theoretical_loss": 3.9064184994563713, + "tokens_seen": 510132224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042697091273821464, + "loss": 2.8344, + "theoretical_loss": 3.9063646361474893, + "tokens_seen": 510197760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269608826479439, + "loss": 2.9352, + "theoretical_loss": 3.906310781694017, + "tokens_seen": 510263296 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426950852557673, + "loss": 2.9629, + "theoretical_loss": 3.9062569360933623, + "tokens_seen": 510328832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042694082246740224, + "loss": 2.9865, + "theoretical_loss": 3.906203099342933, + "tokens_seen": 510394368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269307923771314, + "loss": 2.8716, + "theoretical_loss": 3.9061492714401385, + "tokens_seen": 510459904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269207622868606, + "loss": 2.8544, + "theoretical_loss": 3.9060954523823885, + "tokens_seen": 510525440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269107321965898, + "loss": 3.1565, + "theoretical_loss": 3.906041642167095, + "tokens_seen": 510590976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042690070210631896, + "loss": 2.9906, + "theoretical_loss": 3.9059878407916706, + "tokens_seen": 510656512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042689067201604814, + "loss": 2.8862, + "theoretical_loss": 3.9059340482535285, + "tokens_seen": 510722048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268806419257774, + "loss": 2.815, + "theoretical_loss": 3.905880264550083, + "tokens_seen": 510787584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268706118355065, + "loss": 2.8911, + "theoretical_loss": 3.905826489678751, + "tokens_seen": 510853120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042686058174523574, + "loss": 2.9307, + "theoretical_loss": 3.9057727236369484, + "tokens_seen": 510918656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042685055165496487, + "loss": 2.6768, + "theoretical_loss": 3.9057189664220937, + "tokens_seen": 510984192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268405215646941, + "loss": 2.9439, + "theoretical_loss": 3.905665218031606, + "tokens_seen": 511049728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268304914744233, + "loss": 2.9301, + "theoretical_loss": 3.9056114784629052, + "tokens_seen": 511115264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 292775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.923103094100952, + "objective/train/theoretical_loss": 3.905557747713412, + "objective/train/tokens_used": 531640800, + "theoretical_loss": 3.905557747713412, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042682046138415246, + "loss": 2.9187, + "theoretical_loss": 3.905557747713412, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042681043129388165, + "loss": 3.001, + "theoretical_loss": 3.9055040257805493, + "tokens_seen": 511246336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268004012036109, + "loss": 2.946, + "theoretical_loss": 3.9054503126617406, + "tokens_seen": 511311872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042679037111334, + "loss": 2.8961, + "theoretical_loss": 3.90539660835441, + "tokens_seen": 511377408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042678034102306924, + "loss": 2.9013, + "theoretical_loss": 3.905342912855983, + "tokens_seen": 511442944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042677031093279837, + "loss": 2.8939, + "theoretical_loss": 3.9052892261638865, + "tokens_seen": 511508480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267602808425276, + "loss": 3.2111, + "theoretical_loss": 3.905235548275548, + "tokens_seen": 511574016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267502507522568, + "loss": 2.9137, + "theoretical_loss": 3.9051818791883965, + "tokens_seen": 511639552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042674022066198597, + "loss": 2.8786, + "theoretical_loss": 3.9051282188998613, + "tokens_seen": 511705088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042673019057171515, + "loss": 2.8007, + "theoretical_loss": 3.9050745674073744, + "tokens_seen": 511770624 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042672016048144433, + "loss": 2.7962, + "theoretical_loss": 3.9050209247083663, + "tokens_seen": 511836160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267101303911735, + "loss": 2.9068, + "theoretical_loss": 3.9049672908002715, + "tokens_seen": 511901696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042670010030090275, + "loss": 3.034, + "theoretical_loss": 3.9049136656805237, + "tokens_seen": 511967232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266900702106319, + "loss": 2.7779, + "theoretical_loss": 3.904860049346558, + "tokens_seen": 512032768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266800401203611, + "loss": 2.7907, + "theoretical_loss": 3.9048064417958104, + "tokens_seen": 512098304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042667001003009024, + "loss": 3.0229, + "theoretical_loss": 3.9047528430257192, + "tokens_seen": 512163840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042665997993981947, + "loss": 2.7588, + "theoretical_loss": 3.904699253033723, + "tokens_seen": 512229376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042664994984954865, + "loss": 2.9785, + "theoretical_loss": 3.9046456718172595, + "tokens_seen": 512294912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042663991975927783, + "loss": 2.7284, + "theoretical_loss": 3.9045920993737715, + "tokens_seen": 512360448 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426629889669007, + "loss": 2.9626, + "theoretical_loss": 3.9045385357006994, + "tokens_seen": 512425984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042661985957873625, + "loss": 2.9813, + "theoretical_loss": 3.9044849807954867, + "tokens_seen": 512491520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266098294884654, + "loss": 2.6292, + "theoretical_loss": 3.904431434655576, + "tokens_seen": 512557056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265997993981946, + "loss": 2.8153, + "theoretical_loss": 3.9043778972784144, + "tokens_seen": 512622592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042658976930792374, + "loss": 2.8765, + "theoretical_loss": 3.904324368661446, + "tokens_seen": 512688128 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426579739217653, + "loss": 2.8445, + "theoretical_loss": 3.904270848802119, + "tokens_seen": 512753664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 293228, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3358237743377686, + "objective/train/theoretical_loss": 3.904217337697881, + "objective/train/tokens_used": 533279200, + "theoretical_loss": 3.904217337697881, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042656970912738216, + "loss": 2.832, + "theoretical_loss": 3.904217337697881, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042655967903711134, + "loss": 2.9333, + "theoretical_loss": 3.904163835346181, + "tokens_seen": 512884736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265496489468405, + "loss": 3.0451, + "theoretical_loss": 3.9041103417444694, + "tokens_seen": 512950272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265396188565697, + "loss": 2.9155, + "theoretical_loss": 3.9040568568901977, + "tokens_seen": 513015808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265295887662989, + "loss": 2.9664, + "theoretical_loss": 3.9040033807808188, + "tokens_seen": 513081344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265195586760281, + "loss": 3.024, + "theoretical_loss": 3.9039499134137854, + "tokens_seen": 513146880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042650952858575724, + "loss": 2.8769, + "theoretical_loss": 3.903896454786552, + "tokens_seen": 513212416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264994984954865, + "loss": 2.8886, + "theoretical_loss": 3.903843004896575, + "tokens_seen": 513277952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264894684052156, + "loss": 2.8289, + "theoretical_loss": 3.9037895637413103, + "tokens_seen": 513343488 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042647943831494484, + "loss": 2.8079, + "theoretical_loss": 3.903736131318216, + "tokens_seen": 513409024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426469408224674, + "loss": 2.8159, + "theoretical_loss": 3.9036827076247516, + "tokens_seen": 513474560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264593781344032, + "loss": 2.9007, + "theoretical_loss": 3.9036292926583753, + "tokens_seen": 513540096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264493480441324, + "loss": 2.8079, + "theoretical_loss": 3.903575886416549, + "tokens_seen": 513605632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264393179538616, + "loss": 2.7109, + "theoretical_loss": 3.9035224888967344, + "tokens_seen": 513671168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042642928786359075, + "loss": 2.8299, + "theoretical_loss": 3.9034691000963955, + "tokens_seen": 513736704 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042641925777332, + "loss": 2.8933, + "theoretical_loss": 3.9034157200129953, + "tokens_seen": 513802240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042640922768304916, + "loss": 2.8528, + "theoretical_loss": 3.9033623486439994, + "tokens_seen": 513867776 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042639919759277834, + "loss": 3.0509, + "theoretical_loss": 3.9033089859868744, + "tokens_seen": 513933312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263891675025076, + "loss": 2.8992, + "theoretical_loss": 3.9032556320390874, + "tokens_seen": 513998848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263791374122367, + "loss": 2.7701, + "theoretical_loss": 3.9032022867981055, + "tokens_seen": 514064384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042636910732196594, + "loss": 2.9763, + "theoretical_loss": 3.9031489502614, + "tokens_seen": 514129920 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042635907723169507, + "loss": 2.6713, + "theoretical_loss": 3.9030956224264406, + "tokens_seen": 514195456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263490471414243, + "loss": 2.7494, + "theoretical_loss": 3.903042303290699, + "tokens_seen": 514260992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263390170511535, + "loss": 2.9207, + "theoretical_loss": 3.9029889928516472, + "tokens_seen": 514326528 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042632898696088266, + "loss": 2.804, + "theoretical_loss": 3.902935691106759, + "tokens_seen": 514392064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 294365, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.787262201309204, + "objective/train/theoretical_loss": 3.90288239805351, + "objective/train/tokens_used": 534917600, + "theoretical_loss": 3.90288239805351, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042631895687061185, + "loss": 3.0647, + "theoretical_loss": 3.90288239805351, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263089267803411, + "loss": 2.6791, + "theoretical_loss": 3.902829113689375, + "tokens_seen": 514523136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262988966900702, + "loss": 2.7555, + "theoretical_loss": 3.902775838011831, + "tokens_seen": 514588672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042628886659979944, + "loss": 2.9535, + "theoretical_loss": 3.902722571018356, + "tokens_seen": 514654208 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042627883650952857, + "loss": 2.8113, + "theoretical_loss": 3.902669312706429, + "tokens_seen": 514719744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262688064192578, + "loss": 2.8344, + "theoretical_loss": 3.90261606307353, + "tokens_seen": 514785280 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426258776328987, + "loss": 3.0159, + "theoretical_loss": 3.90256282211714, + "tokens_seen": 514850816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042624874623871617, + "loss": 2.8466, + "theoretical_loss": 3.9025095898347404, + "tokens_seen": 514916352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042623871614844535, + "loss": 2.6467, + "theoretical_loss": 3.9024563662238156, + "tokens_seen": 514981888 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042622868605817453, + "loss": 2.5978, + "theoretical_loss": 3.902403151281848, + "tokens_seen": 515047424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262186559679037, + "loss": 2.5738, + "theoretical_loss": 3.902349945006325, + "tokens_seen": 515112960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042620862587763295, + "loss": 3.0073, + "theoretical_loss": 3.9022967473947308, + "tokens_seen": 515178496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261985957873621, + "loss": 2.8529, + "theoretical_loss": 3.9022435584445545, + "tokens_seen": 515244032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261885656970913, + "loss": 2.8852, + "theoretical_loss": 3.902190378153283, + "tokens_seen": 515309568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042617853560682044, + "loss": 2.8072, + "theoretical_loss": 3.9021372065184066, + "tokens_seen": 515375104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042616850551654967, + "loss": 2.724, + "theoretical_loss": 3.902084043537416, + "tokens_seen": 515440640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042615847542627885, + "loss": 2.8464, + "theoretical_loss": 3.902030889207802, + "tokens_seen": 515506176 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042614844533600803, + "loss": 2.5744, + "theoretical_loss": 3.9019777435270573, + "tokens_seen": 515571712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261384152457372, + "loss": 2.8335, + "theoretical_loss": 3.901924606492675, + "tokens_seen": 515637248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042612838515546645, + "loss": 3.0876, + "theoretical_loss": 3.9018714781021515, + "tokens_seen": 515702784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261183550651956, + "loss": 2.8044, + "theoretical_loss": 3.9018183583529806, + "tokens_seen": 515768320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261083249749248, + "loss": 2.9733, + "theoretical_loss": 3.9017652472426603, + "tokens_seen": 515833856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042609829488465394, + "loss": 2.9147, + "theoretical_loss": 3.9017121447686876, + "tokens_seen": 515899392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260882647943832, + "loss": 2.93, + "theoretical_loss": 3.9016590509285614, + "tokens_seen": 515964928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042607823470411236, + "loss": 2.8142, + "theoretical_loss": 3.901605965719782, + "tokens_seen": 516030464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 295169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.053830623626709, + "objective/train/theoretical_loss": 3.9015528891398494, + "objective/train/tokens_used": 536556000, + "theoretical_loss": 3.9015528891398494, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042606820461384154, + "loss": 3.1288, + "theoretical_loss": 3.9015528891398494, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260581745235707, + "loss": 2.694, + "theoretical_loss": 3.9014998211862673, + "tokens_seen": 516161536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260481444332999, + "loss": 3.0028, + "theoretical_loss": 3.901446761856537, + "tokens_seen": 516227072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260381143430291, + "loss": 2.7803, + "theoretical_loss": 3.901393711148163, + "tokens_seen": 516292608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260280842527583, + "loss": 2.8191, + "theoretical_loss": 3.901340669058651, + "tokens_seen": 516358144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042601805416248744, + "loss": 3.0177, + "theoretical_loss": 3.9012876355855064, + "tokens_seen": 516423680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260080240722167, + "loss": 2.7097, + "theoretical_loss": 3.9012346107262363, + "tokens_seen": 516489216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259979939819458, + "loss": 2.9959, + "theoretical_loss": 3.9011815944783494, + "tokens_seen": 516554752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042598796389167504, + "loss": 3.0318, + "theoretical_loss": 3.9011285868393544, + "tokens_seen": 516620288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259779338014042, + "loss": 2.9474, + "theoretical_loss": 3.901075587806762, + "tokens_seen": 516685824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259679037111334, + "loss": 3.0822, + "theoretical_loss": 3.901022597378083, + "tokens_seen": 516751360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259578736208626, + "loss": 2.8803, + "theoretical_loss": 3.90096961555083, + "tokens_seen": 516816896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259478435305918, + "loss": 2.8745, + "theoretical_loss": 3.9009166423225166, + "tokens_seen": 516882432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042593781344032095, + "loss": 3.0311, + "theoretical_loss": 3.9008636776906567, + "tokens_seen": 516947968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259277833500502, + "loss": 2.9673, + "theoretical_loss": 3.9008107216527663, + "tokens_seen": 517013504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259177532597793, + "loss": 2.9108, + "theoretical_loss": 3.9007577742063613, + "tokens_seen": 517079040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042590772316950854, + "loss": 2.7388, + "theoretical_loss": 3.900704835348959, + "tokens_seen": 517144576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258976930792377, + "loss": 2.991, + "theoretical_loss": 3.900651905078079, + "tokens_seen": 517210112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258876629889669, + "loss": 2.7754, + "theoretical_loss": 3.90059898339124, + "tokens_seen": 517275648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258776328986961, + "loss": 2.8331, + "theoretical_loss": 3.9005460702859627, + "tokens_seen": 517341184 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042586760280842527, + "loss": 2.9162, + "theoretical_loss": 3.9004931657597686, + "tokens_seen": 517406720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042585757271815445, + "loss": 2.8646, + "theoretical_loss": 3.900440269810181, + "tokens_seen": 517472256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258475426278837, + "loss": 3.0237, + "theoretical_loss": 3.900387382434723, + "tokens_seen": 517537792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258375125376128, + "loss": 2.8647, + "theoretical_loss": 3.900334503630919, + "tokens_seen": 517603328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042582748244734205, + "loss": 2.8821, + "theoretical_loss": 3.9002816333962955, + "tokens_seen": 517668864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 295690, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6655638217926025, + "objective/train/theoretical_loss": 3.9002287717283783, + "objective/train/tokens_used": 538194400, + "theoretical_loss": 3.9002287717283783, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258174523570712, + "loss": 2.6781, + "theoretical_loss": 3.9002287717283783, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258074222668004, + "loss": 2.737, + "theoretical_loss": 3.9001759186246967, + "tokens_seen": 517799936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257973921765296, + "loss": 2.8423, + "theoretical_loss": 3.900123074082778, + "tokens_seen": 517865472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042578736208625877, + "loss": 3.0461, + "theoretical_loss": 3.9000702381001533, + "tokens_seen": 517931008 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042577733199598795, + "loss": 3.0473, + "theoretical_loss": 3.900017410674352, + "tokens_seen": 517996544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257673019057172, + "loss": 2.764, + "theoretical_loss": 3.899964591802907, + "tokens_seen": 518062080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257572718154463, + "loss": 3.0369, + "theoretical_loss": 3.8999117814833513, + "tokens_seen": 518127616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042574724172517555, + "loss": 2.8756, + "theoretical_loss": 3.899858979713218, + "tokens_seen": 518193152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257372116349047, + "loss": 3.0317, + "theoretical_loss": 3.899806186490043, + "tokens_seen": 518258688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257271815446339, + "loss": 2.885, + "theoretical_loss": 3.8997534018113615, + "tokens_seen": 518324224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257171514543631, + "loss": 2.8826, + "theoretical_loss": 3.8997006256747113, + "tokens_seen": 518389760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257071213640923, + "loss": 2.9002, + "theoretical_loss": 3.8996478580776293, + "tokens_seen": 518455296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042569709127382145, + "loss": 2.8571, + "theoretical_loss": 3.899595099017656, + "tokens_seen": 518520832 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042568706118355064, + "loss": 2.8265, + "theoretical_loss": 3.8995423484923304, + "tokens_seen": 518586368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256770310932798, + "loss": 2.7872, + "theoretical_loss": 3.899489606499193, + "tokens_seen": 518651904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042566700100300905, + "loss": 2.8207, + "theoretical_loss": 3.8994368730357873, + "tokens_seen": 518717440 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042565697091273823, + "loss": 2.8399, + "theoretical_loss": 3.8993841480996565, + "tokens_seen": 518782976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256469408224674, + "loss": 2.8967, + "theoretical_loss": 3.899331431688344, + "tokens_seen": 518848512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042563691073219665, + "loss": 2.8206, + "theoretical_loss": 3.8992787237993944, + "tokens_seen": 518914048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256268806419258, + "loss": 2.6858, + "theoretical_loss": 3.8992260244303547, + "tokens_seen": 518979584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425616850551655, + "loss": 2.8494, + "theoretical_loss": 3.899173333578772, + "tokens_seen": 519045120 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042560682046138414, + "loss": 2.7067, + "theoretical_loss": 3.8991206512421943, + "tokens_seen": 519110656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255967903711134, + "loss": 2.8115, + "theoretical_loss": 3.8990679774181713, + "tokens_seen": 519176192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042558676028084256, + "loss": 2.8671, + "theoretical_loss": 3.899015312104253, + "tokens_seen": 519241728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042557673019057174, + "loss": 2.787, + "theoretical_loss": 3.89896265529799, + "tokens_seen": 519307264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 296800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2833645343780518, + "objective/train/theoretical_loss": 3.8989100069969354, + "objective/train/tokens_used": 539832800, + "theoretical_loss": 3.8989100069969354, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255667001003009, + "loss": 2.8438, + "theoretical_loss": 3.8989100069969354, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255566700100301, + "loss": 2.7813, + "theoretical_loss": 3.8988573671986417, + "tokens_seen": 519438336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255466399197593, + "loss": 2.6913, + "theoretical_loss": 3.898804735900664, + "tokens_seen": 519503872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255366098294885, + "loss": 2.9325, + "theoretical_loss": 3.8987521131005574, + "tokens_seen": 519569408 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042552657973921764, + "loss": 3.0525, + "theoretical_loss": 3.898699498795878, + "tokens_seen": 519634944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255165496489469, + "loss": 3.1233, + "theoretical_loss": 3.898646892984183, + "tokens_seen": 519700480 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425506519558676, + "loss": 2.9536, + "theoretical_loss": 3.8985942956630306, + "tokens_seen": 519766016 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549648946840524, + "loss": 2.8703, + "theoretical_loss": 3.898541706829981, + "tokens_seen": 519831552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254864593781344, + "loss": 3.0978, + "theoretical_loss": 3.898489126482594, + "tokens_seen": 519897088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254764292878636, + "loss": 3.0565, + "theoretical_loss": 3.8984365546184305, + "tokens_seen": 519962624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254663991975928, + "loss": 3.0101, + "theoretical_loss": 3.898383991235053, + "tokens_seen": 520028160 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425456369107322, + "loss": 3.0552, + "theoretical_loss": 3.898331436330026, + "tokens_seen": 520093696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042544633901705115, + "loss": 2.6484, + "theoretical_loss": 3.8982788899009124, + "tokens_seen": 520159232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254363089267804, + "loss": 2.9835, + "theoretical_loss": 3.8982263519452793, + "tokens_seen": 520224768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254262788365095, + "loss": 3.0324, + "theoretical_loss": 3.8981738224606914, + "tokens_seen": 520290304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042541624874623874, + "loss": 2.9913, + "theoretical_loss": 3.898121301444716, + "tokens_seen": 520355840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254062186559679, + "loss": 2.8293, + "theoretical_loss": 3.898068788894923, + "tokens_seen": 520421376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253961885656971, + "loss": 3.0361, + "theoretical_loss": 3.898016284808881, + "tokens_seen": 520486912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253861584754263, + "loss": 2.9718, + "theoretical_loss": 3.8979637891841605, + "tokens_seen": 520552448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042537612838515547, + "loss": 2.8781, + "theoretical_loss": 3.8979113020183327, + "tokens_seen": 520617984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042536609829488465, + "loss": 2.9975, + "theoretical_loss": 3.8978588233089706, + "tokens_seen": 520683520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253560682046139, + "loss": 2.7801, + "theoretical_loss": 3.897806353053647, + "tokens_seen": 520749056 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425346038114343, + "loss": 2.9144, + "theoretical_loss": 3.897753891249936, + "tokens_seen": 520814592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533600802407225, + "loss": 3.0736, + "theoretical_loss": 3.897701437895414, + "tokens_seen": 520880128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253259779338014, + "loss": 2.9123, + "theoretical_loss": 3.8976489929876568, + "tokens_seen": 520945664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 297883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9455604553222656, + "objective/train/theoretical_loss": 3.8975965565242423, + "objective/train/tokens_used": 541471200, + "theoretical_loss": 3.8975965565242423, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253159478435306, + "loss": 2.724, + "theoretical_loss": 3.8975965565242423, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253059177532598, + "loss": 3.0535, + "theoretical_loss": 3.897544128502749, + "tokens_seen": 521076736 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042529588766298897, + "loss": 2.9999, + "theoretical_loss": 3.897491708920755, + "tokens_seen": 521142272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042528585757271815, + "loss": 2.9009, + "theoretical_loss": 3.897439297775842, + "tokens_seen": 521207808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252758274824474, + "loss": 2.8293, + "theoretical_loss": 3.8973868950655914, + "tokens_seen": 521273344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252657973921765, + "loss": 2.9852, + "theoretical_loss": 3.8973345007875846, + "tokens_seen": 521338880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042525576730190575, + "loss": 2.9392, + "theoretical_loss": 3.897282114939406, + "tokens_seen": 521404416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252457372116349, + "loss": 2.8227, + "theoretical_loss": 3.89722973751864, + "tokens_seen": 521469952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252357071213641, + "loss": 2.7058, + "theoretical_loss": 3.897177368522872, + "tokens_seen": 521535488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252256770310933, + "loss": 2.9889, + "theoretical_loss": 3.897125007949687, + "tokens_seen": 521601024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252156469408225, + "loss": 2.935, + "theoretical_loss": 3.8970726557966744, + "tokens_seen": 521666560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042520561685055165, + "loss": 2.9667, + "theoretical_loss": 3.897020312061421, + "tokens_seen": 521732096 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042519558676028084, + "loss": 2.941, + "theoretical_loss": 3.896967976741517, + "tokens_seen": 521797632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042518555667001, + "loss": 2.9092, + "theoretical_loss": 3.896915649834553, + "tokens_seen": 521863168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042517552657973925, + "loss": 2.6848, + "theoretical_loss": 3.8968633313381202, + "tokens_seen": 521928704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251654964894684, + "loss": 2.7817, + "theoretical_loss": 3.8968110212498104, + "tokens_seen": 521994240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251554663991976, + "loss": 2.7736, + "theoretical_loss": 3.8967587195672175, + "tokens_seen": 522059776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251454363089268, + "loss": 2.9141, + "theoretical_loss": 3.8967064262879356, + "tokens_seen": 522125312 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425135406218656, + "loss": 2.6049, + "theoretical_loss": 3.8966541414095595, + "tokens_seen": 522190848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042512537612838516, + "loss": 2.6158, + "theoretical_loss": 3.8966018649296865, + "tokens_seen": 522256384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042511534603811434, + "loss": 2.7402, + "theoretical_loss": 3.8965495968459134, + "tokens_seen": 522321920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251053159478435, + "loss": 3.1019, + "theoretical_loss": 3.896497337155839, + "tokens_seen": 522387456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042509528585757276, + "loss": 2.9382, + "theoretical_loss": 3.896445085857062, + "tokens_seen": 522452992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250852557673019, + "loss": 2.9426, + "theoretical_loss": 3.896392842947183, + "tokens_seen": 522518528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250752256770311, + "loss": 2.6713, + "theoretical_loss": 3.8963406084238024, + "tokens_seen": 522584064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 298609, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9465911388397217, + "objective/train/theoretical_loss": 3.8962883822845233, + "objective/train/tokens_used": 543109600, + "theoretical_loss": 3.8962883822845233, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042506519558676024, + "loss": 2.7065, + "theoretical_loss": 3.8962883822845233, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250551654964895, + "loss": 2.9139, + "theoretical_loss": 3.8962361645269494, + "tokens_seen": 522715136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042504513540621866, + "loss": 2.9572, + "theoretical_loss": 3.8961839551486843, + "tokens_seen": 522780672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042503510531594784, + "loss": 2.8489, + "theoretical_loss": 3.8961317541473326, + "tokens_seen": 522846208 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425025075225677, + "loss": 3.0263, + "theoretical_loss": 3.8960795615205015, + "tokens_seen": 522911744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250150451354062, + "loss": 3.0704, + "theoretical_loss": 3.896027377265798, + "tokens_seen": 522977280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250050150451354, + "loss": 2.9052, + "theoretical_loss": 3.8959752013808293, + "tokens_seen": 523042816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249949849548646, + "loss": 2.916, + "theoretical_loss": 3.895923033863206, + "tokens_seen": 523108352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042498495486459375, + "loss": 2.955, + "theoretical_loss": 3.8958708747105373, + "tokens_seen": 523173888 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424974924774323, + "loss": 2.8302, + "theoretical_loss": 3.895818723920434, + "tokens_seen": 523239424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042496489468405216, + "loss": 2.7975, + "theoretical_loss": 3.895766581490509, + "tokens_seen": 523304960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042495486459378135, + "loss": 2.8087, + "theoretical_loss": 3.895714447418375, + "tokens_seen": 523370496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249448345035105, + "loss": 2.9109, + "theoretical_loss": 3.8956623217016464, + "tokens_seen": 523436032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249348044132397, + "loss": 2.8087, + "theoretical_loss": 3.895610204337938, + "tokens_seen": 523501568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042492477432296894, + "loss": 2.8582, + "theoretical_loss": 3.8955580953248656, + "tokens_seen": 523567104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249147442326981, + "loss": 2.8843, + "theoretical_loss": 3.895505994660046, + "tokens_seen": 523632640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249047141424273, + "loss": 3.1563, + "theoretical_loss": 3.8954539023410972, + "tokens_seen": 523698176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248946840521565, + "loss": 2.9491, + "theoretical_loss": 3.895401818365639, + "tokens_seen": 523763712 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042488465396188567, + "loss": 2.9519, + "theoretical_loss": 3.89534974273129, + "tokens_seen": 523829248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042487462387161485, + "loss": 2.5973, + "theoretical_loss": 3.895297675435673, + "tokens_seen": 523894784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248645937813441, + "loss": 2.8273, + "theoretical_loss": 3.8952456164764078, + "tokens_seen": 523960320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248545636910732, + "loss": 2.7711, + "theoretical_loss": 3.8951935658511183, + "tokens_seen": 524025856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042484453360080245, + "loss": 2.9623, + "theoretical_loss": 3.895141523557428, + "tokens_seen": 524091392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248345035105316, + "loss": 2.7556, + "theoretical_loss": 3.895089489592962, + "tokens_seen": 524156928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248244734202608, + "loss": 2.907, + "theoretical_loss": 3.8950374639553456, + "tokens_seen": 524222464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 299683, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.230189085006714, + "objective/train/theoretical_loss": 3.894985446642206, + "objective/train/tokens_used": 544748000, + "theoretical_loss": 3.894985446642206, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042481444332999, + "loss": 2.674, + "theoretical_loss": 3.894985446642206, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042480441323971917, + "loss": 2.7178, + "theoretical_loss": 3.8949334376511713, + "tokens_seen": 524353536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042479438314944835, + "loss": 2.6602, + "theoretical_loss": 3.894881436979869, + "tokens_seen": 524419072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247843530591776, + "loss": 2.881, + "theoretical_loss": 3.8948294446259295, + "tokens_seen": 524484608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247743229689067, + "loss": 3.029, + "theoretical_loss": 3.894777460586983, + "tokens_seen": 524550144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042476429287863595, + "loss": 3.0115, + "theoretical_loss": 3.894725484860662, + "tokens_seen": 524615680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247542627883651, + "loss": 2.9568, + "theoretical_loss": 3.8946735174445983, + "tokens_seen": 524681216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247442326980943, + "loss": 2.6509, + "theoretical_loss": 3.894621558336426, + "tokens_seen": 524746752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247342026078235, + "loss": 2.7833, + "theoretical_loss": 3.8945696075337786, + "tokens_seen": 524812288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247241725175527, + "loss": 2.8974, + "theoretical_loss": 3.8945176650342925, + "tokens_seen": 524877824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042471414242728186, + "loss": 3.0308, + "theoretical_loss": 3.894465730835604, + "tokens_seen": 524943360 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042470411233701104, + "loss": 2.8243, + "theoretical_loss": 3.8944138049353505, + "tokens_seen": 525008896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246940822467402, + "loss": 2.8344, + "theoretical_loss": 3.8943618873311703, + "tokens_seen": 525074432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042468405215646945, + "loss": 2.6421, + "theoretical_loss": 3.8943099780207024, + "tokens_seen": 525139968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246740220661986, + "loss": 3.2505, + "theoretical_loss": 3.894258077001587, + "tokens_seen": 525205504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246639919759278, + "loss": 2.9114, + "theoretical_loss": 3.8942061842714666, + "tokens_seen": 525271040 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424653961885657, + "loss": 2.9455, + "theoretical_loss": 3.8941542998279823, + "tokens_seen": 525336576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246439317953862, + "loss": 2.7369, + "theoretical_loss": 3.894102423668778, + "tokens_seen": 525402112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042463390170511536, + "loss": 2.879, + "theoretical_loss": 3.8940505557914973, + "tokens_seen": 525467648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042462387161484454, + "loss": 2.9338, + "theoretical_loss": 3.893998696193785, + "tokens_seen": 525533184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246138415245737, + "loss": 2.7185, + "theoretical_loss": 3.8939468448732892, + "tokens_seen": 525598720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042460381143430296, + "loss": 2.723, + "theoretical_loss": 3.893895001827654, + "tokens_seen": 525664256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245937813440321, + "loss": 2.8829, + "theoretical_loss": 3.89384316705453, + "tokens_seen": 525729792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245837512537613, + "loss": 2.8745, + "theoretical_loss": 3.893791340551565, + "tokens_seen": 525795328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042457372116349044, + "loss": 2.9526, + "theoretical_loss": 3.8937395223164093, + "tokens_seen": 525860864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 300593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.640450954437256, + "objective/train/theoretical_loss": 3.893687712346713, + "objective/train/tokens_used": 546386400, + "theoretical_loss": 3.893687712346713, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245636910732197, + "loss": 3.066, + "theoretical_loss": 3.893687712346713, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042455366098294886, + "loss": 2.814, + "theoretical_loss": 3.8936359106401293, + "tokens_seen": 525991936 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042454363089267804, + "loss": 2.7216, + "theoretical_loss": 3.8935841171943095, + "tokens_seen": 526057472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245336008024072, + "loss": 2.7412, + "theoretical_loss": 3.893532332006909, + "tokens_seen": 526123008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245235707121364, + "loss": 2.9448, + "theoretical_loss": 3.8934805550755813, + "tokens_seen": 526188544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245135406218656, + "loss": 2.7285, + "theoretical_loss": 3.893428786397983, + "tokens_seen": 526254080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245035105315948, + "loss": 2.7658, + "theoretical_loss": 3.8933770259717697, + "tokens_seen": 526319616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042449348044132395, + "loss": 3.0131, + "theoretical_loss": 3.8933252737946002, + "tokens_seen": 526385152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244834503510532, + "loss": 2.7621, + "theoretical_loss": 3.893273529864132, + "tokens_seen": 526450688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042447342026078236, + "loss": 2.893, + "theoretical_loss": 3.893221794178025, + "tokens_seen": 526516224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042446339017051155, + "loss": 2.8413, + "theoretical_loss": 3.893170066733941, + "tokens_seen": 526581760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244533600802407, + "loss": 2.6966, + "theoretical_loss": 3.8931183475295397, + "tokens_seen": 526647296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244433299899699, + "loss": 2.6659, + "theoretical_loss": 3.8930666365624838, + "tokens_seen": 526712832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244332998996991, + "loss": 2.8686, + "theoretical_loss": 3.893014933830437, + "tokens_seen": 526778368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244232698094283, + "loss": 2.7931, + "theoretical_loss": 3.8929632393310634, + "tokens_seen": 526843904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042441323971915745, + "loss": 2.9316, + "theoretical_loss": 3.892911553062029, + "tokens_seen": 526909440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244032096288867, + "loss": 2.7593, + "theoretical_loss": 3.892859875020999, + "tokens_seen": 526974976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243931795386158, + "loss": 2.6133, + "theoretical_loss": 3.8928082052056405, + "tokens_seen": 527040512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042438314944834505, + "loss": 3.0805, + "theoretical_loss": 3.892756543613623, + "tokens_seen": 527106048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042437311935807423, + "loss": 3.0741, + "theoretical_loss": 3.892704890242614, + "tokens_seen": 527171584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243630892678034, + "loss": 2.9281, + "theoretical_loss": 3.892653245090284, + "tokens_seen": 527237120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243530591775326, + "loss": 2.9436, + "theoretical_loss": 3.8926016081543047, + "tokens_seen": 527302656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243430290872618, + "loss": 2.8254, + "theoretical_loss": 3.892549979432347, + "tokens_seen": 527368192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042433299899699095, + "loss": 2.8296, + "theoretical_loss": 3.8924983589220843, + "tokens_seen": 527433728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243229689067202, + "loss": 2.8305, + "theoretical_loss": 3.8924467466211903, + "tokens_seen": 527499264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 301147, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.089402198791504, + "objective/train/theoretical_loss": 3.8923951425273398, + "objective/train/tokens_used": 548024800, + "theoretical_loss": 3.8923951425273398, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243129388164493, + "loss": 2.883, + "theoretical_loss": 3.8923951425273398, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042430290872617855, + "loss": 2.8607, + "theoretical_loss": 3.8923435466382084, + "tokens_seen": 527630336 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042429287863590773, + "loss": 2.9292, + "theoretical_loss": 3.892291958951473, + "tokens_seen": 527695872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242828485456369, + "loss": 2.8227, + "theoretical_loss": 3.892240379464811, + "tokens_seen": 527761408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242728184553661, + "loss": 2.941, + "theoretical_loss": 3.8921888081759004, + "tokens_seen": 527826944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242627883650953, + "loss": 2.7021, + "theoretical_loss": 3.892137245082422, + "tokens_seen": 527892480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042425275827482446, + "loss": 2.98, + "theoretical_loss": 3.8920856901820553, + "tokens_seen": 527958016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242427281845537, + "loss": 2.8898, + "theoretical_loss": 3.8920341434724817, + "tokens_seen": 528023552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242326980942828, + "loss": 2.8186, + "theoretical_loss": 3.8919826049513837, + "tokens_seen": 528089088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042422266800401206, + "loss": 2.9792, + "theoretical_loss": 3.891931074616444, + "tokens_seen": 528154624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242126379137412, + "loss": 3.1397, + "theoretical_loss": 3.8918795524653484, + "tokens_seen": 528220160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242026078234704, + "loss": 2.5991, + "theoretical_loss": 3.89182803849578, + "tokens_seen": 528285696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241925777331996, + "loss": 2.9587, + "theoretical_loss": 3.8917765327054266, + "tokens_seen": 528351232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241825476429288, + "loss": 2.8182, + "theoretical_loss": 3.8917250350919748, + "tokens_seen": 528416768 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424172517552658, + "loss": 2.8705, + "theoretical_loss": 3.891673545653112, + "tokens_seen": 528482304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241624874623872, + "loss": 2.6681, + "theoretical_loss": 3.891622064386527, + "tokens_seen": 528547840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241524573721164, + "loss": 2.9959, + "theoretical_loss": 3.89157059128991, + "tokens_seen": 528613376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042414242728184556, + "loss": 2.9152, + "theoretical_loss": 3.8915191263609525, + "tokens_seen": 528678912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042413239719157474, + "loss": 2.9309, + "theoretical_loss": 3.8914676695973456, + "tokens_seen": 528744448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241223671013039, + "loss": 2.8499, + "theoretical_loss": 3.8914162209967813, + "tokens_seen": 528809984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042411233701103316, + "loss": 2.915, + "theoretical_loss": 3.8913647805569545, + "tokens_seen": 528875520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241023069207623, + "loss": 2.9174, + "theoretical_loss": 3.891313348275559, + "tokens_seen": 528941056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240922768304915, + "loss": 2.9057, + "theoretical_loss": 3.8912619241502906, + "tokens_seen": 529006592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042408224674022065, + "loss": 2.674, + "theoretical_loss": 3.891210508178845, + "tokens_seen": 529072128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240722166499499, + "loss": 2.7977, + "theoretical_loss": 3.8911591003589203, + "tokens_seen": 529137664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 301749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2091262340545654, + "objective/train/theoretical_loss": 3.891107700688215, + "objective/train/tokens_used": 549663200, + "theoretical_loss": 3.891107700688215, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042406218655967906, + "loss": 2.9569, + "theoretical_loss": 3.891107700688215, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042405215646940824, + "loss": 2.9637, + "theoretical_loss": 3.8910563091644277, + "tokens_seen": 529268736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240421263791374, + "loss": 2.8253, + "theoretical_loss": 3.891004925785259, + "tokens_seen": 529334272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240320962888666, + "loss": 2.7514, + "theoretical_loss": 3.890953550548409, + "tokens_seen": 529399808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240220661985958, + "loss": 2.8764, + "theoretical_loss": 3.8909021834515816, + "tokens_seen": 529465344 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424012036108325, + "loss": 2.8901, + "theoretical_loss": 3.8908508244924778, + "tokens_seen": 529530880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042400200601805415, + "loss": 2.7382, + "theoretical_loss": 3.8907994736688027, + "tokens_seen": 529596416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239919759277834, + "loss": 2.7764, + "theoretical_loss": 3.8907481309782606, + "tokens_seen": 529661952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042398194583751256, + "loss": 3.0124, + "theoretical_loss": 3.890696796418558, + "tokens_seen": 529727488 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042397191574724175, + "loss": 2.912, + "theoretical_loss": 3.8906454699874002, + "tokens_seen": 529793024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239618856569709, + "loss": 2.8759, + "theoretical_loss": 3.890594151682496, + "tokens_seen": 529858560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239518555667001, + "loss": 2.779, + "theoretical_loss": 3.8905428415015537, + "tokens_seen": 529924096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239418254764293, + "loss": 2.8285, + "theoretical_loss": 3.8904915394422828, + "tokens_seen": 529989632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239317953861585, + "loss": 2.7266, + "theoretical_loss": 3.8904402455023934, + "tokens_seen": 530055168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042392176529588765, + "loss": 2.9637, + "theoretical_loss": 3.890388959679597, + "tokens_seen": 530120704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239117352056169, + "loss": 2.7824, + "theoretical_loss": 3.8903376819716056, + "tokens_seen": 530186240 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423901705115346, + "loss": 2.9065, + "theoretical_loss": 3.890286412376133, + "tokens_seen": 530251776 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042389167502507525, + "loss": 2.7732, + "theoretical_loss": 3.8902351508908923, + "tokens_seen": 530317312 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042388164493480443, + "loss": 2.7168, + "theoretical_loss": 3.8901838975136, + "tokens_seen": 530382848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238716148445336, + "loss": 2.9481, + "theoretical_loss": 3.890132652241971, + "tokens_seen": 530448384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238615847542628, + "loss": 2.4908, + "theoretical_loss": 3.890081415073723, + "tokens_seen": 530513920 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423851554663992, + "loss": 3.0909, + "theoretical_loss": 3.8900301860065727, + "tokens_seen": 530579456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042384152457372115, + "loss": 2.7948, + "theoretical_loss": 3.8899789650382397, + "tokens_seen": 530644992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238314944834504, + "loss": 2.9124, + "theoretical_loss": 3.889927752166443, + "tokens_seen": 530710528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238214643931795, + "loss": 2.7639, + "theoretical_loss": 3.889876547388904, + "tokens_seen": 530776064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 302930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.119241714477539, + "objective/train/theoretical_loss": 3.889825350703344, + "objective/train/tokens_used": 551301600, + "theoretical_loss": 3.889825350703344, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042381143430290875, + "loss": 2.8456, + "theoretical_loss": 3.889825350703344, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042380140421263793, + "loss": 3.1018, + "theoretical_loss": 3.889774162107485, + "tokens_seen": 530907136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237913741223671, + "loss": 2.637, + "theoretical_loss": 3.889722981599051, + "tokens_seen": 530972672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237813440320963, + "loss": 2.8671, + "theoretical_loss": 3.8896718091757654, + "tokens_seen": 531038208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237713139418255, + "loss": 2.8487, + "theoretical_loss": 3.889620644835354, + "tokens_seen": 531103744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042376128385155466, + "loss": 2.7057, + "theoretical_loss": 3.889569488575543, + "tokens_seen": 531169280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237512537612839, + "loss": 2.6746, + "theoretical_loss": 3.8895183403940594, + "tokens_seen": 531234816 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423741223671013, + "loss": 2.564, + "theoretical_loss": 3.889467200288631, + "tokens_seen": 531300352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042373119358074226, + "loss": 2.684, + "theoretical_loss": 3.889416068256986, + "tokens_seen": 531365888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237211634904714, + "loss": 2.7755, + "theoretical_loss": 3.8893649442968554, + "tokens_seen": 531431424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237111334002006, + "loss": 2.9413, + "theoretical_loss": 3.8893138284059696, + "tokens_seen": 531496960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237011033099298, + "loss": 3.0095, + "theoretical_loss": 3.8892627205820602, + "tokens_seen": 531562496 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423691073219659, + "loss": 2.9443, + "theoretical_loss": 3.8892116208228593, + "tokens_seen": 531628032 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042368104312938816, + "loss": 2.7001, + "theoretical_loss": 3.8891605291261007, + "tokens_seen": 531693568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236710130391174, + "loss": 2.7396, + "theoretical_loss": 3.889109445489519, + "tokens_seen": 531759104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236609829488465, + "loss": 2.8817, + "theoretical_loss": 3.8890583699108494, + "tokens_seen": 531824640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042365095285857576, + "loss": 3.0576, + "theoretical_loss": 3.889007302387828, + "tokens_seen": 531890176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236409227683049, + "loss": 2.9742, + "theoretical_loss": 3.8889562429181916, + "tokens_seen": 531955712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236308926780341, + "loss": 2.8414, + "theoretical_loss": 3.8889051914996786, + "tokens_seen": 532021248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236208625877633, + "loss": 2.9494, + "theoretical_loss": 3.888854148130028, + "tokens_seen": 532086784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236108324974925, + "loss": 2.9868, + "theoretical_loss": 3.88880311280698, + "tokens_seen": 532152320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042360080240722166, + "loss": 2.7983, + "theoretical_loss": 3.8887520855282744, + "tokens_seen": 532217856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042359077231695085, + "loss": 2.9949, + "theoretical_loss": 3.888701066291654, + "tokens_seen": 532283392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042358074222668, + "loss": 2.7991, + "theoretical_loss": 3.888650055094861, + "tokens_seen": 532348928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042357071213640926, + "loss": 2.8885, + "theoretical_loss": 3.888599051935638, + "tokens_seen": 532414464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 303466, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.733795404434204, + "objective/train/theoretical_loss": 3.8885480568117314, + "objective/train/tokens_used": 552940000, + "theoretical_loss": 3.8885480568117314, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235606820461384, + "loss": 2.8752, + "theoretical_loss": 3.8885480568117314, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235506519558676, + "loss": 2.9894, + "theoretical_loss": 3.888497069720885, + "tokens_seen": 532545536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042354062186559675, + "loss": 2.7836, + "theoretical_loss": 3.888446090660845, + "tokens_seen": 532611072 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423530591775326, + "loss": 2.7741, + "theoretical_loss": 3.88839511962936, + "tokens_seen": 532676608 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042352056168505517, + "loss": 3.0174, + "theoretical_loss": 3.8883441566241763, + "tokens_seen": 532742144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042351053159478435, + "loss": 2.9035, + "theoretical_loss": 3.8882932016430436, + "tokens_seen": 532807680 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042350050150451353, + "loss": 2.735, + "theoretical_loss": 3.8882422546837123, + "tokens_seen": 532873216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042349047141424276, + "loss": 2.9197, + "theoretical_loss": 3.888191315743933, + "tokens_seen": 532938752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234804413239719, + "loss": 2.8019, + "theoretical_loss": 3.888140384821457, + "tokens_seen": 533004288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042347041123370113, + "loss": 2.7563, + "theoretical_loss": 3.8880894619140367, + "tokens_seen": 533069824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042346038114343025, + "loss": 3.0713, + "theoretical_loss": 3.888038547019427, + "tokens_seen": 533135360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234503510531595, + "loss": 2.9346, + "theoretical_loss": 3.88798764013538, + "tokens_seen": 533200896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042344032096288867, + "loss": 2.9696, + "theoretical_loss": 3.8879367412596535, + "tokens_seen": 533266432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042343029087261785, + "loss": 2.8591, + "theoretical_loss": 3.887885850390002, + "tokens_seen": 533331968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234202607823471, + "loss": 2.7303, + "theoretical_loss": 3.8878349675241832, + "tokens_seen": 533397504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234102306920762, + "loss": 2.936, + "theoretical_loss": 3.8877840926599556, + "tokens_seen": 533463040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042340020060180545, + "loss": 2.6803, + "theoretical_loss": 3.8877332257950776, + "tokens_seen": 533528576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042339017051153463, + "loss": 2.8323, + "theoretical_loss": 3.887682366927309, + "tokens_seen": 533594112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233801404212638, + "loss": 2.9108, + "theoretical_loss": 3.8876315160544106, + "tokens_seen": 533659648 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423370110330993, + "loss": 2.9117, + "theoretical_loss": 3.8875806731741447, + "tokens_seen": 533725184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233600802407222, + "loss": 2.8684, + "theoretical_loss": 3.887529838284273, + "tokens_seen": 533790720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042335005015045135, + "loss": 2.7934, + "theoretical_loss": 3.887479011382559, + "tokens_seen": 533856256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233400200601806, + "loss": 2.6989, + "theoretical_loss": 3.8874281924667677, + "tokens_seen": 533921792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233299899699097, + "loss": 2.6725, + "theoretical_loss": 3.887377381534664, + "tokens_seen": 533987328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042331995987963895, + "loss": 2.764, + "theoretical_loss": 3.8873265785840134, + "tokens_seen": 534052864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 304686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.990126848220825, + "objective/train/theoretical_loss": 3.8872757836125844, + "objective/train/tokens_used": 554578400, + "theoretical_loss": 3.8872757836125844, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042330992978936813, + "loss": 2.8816, + "theoretical_loss": 3.8872757836125844, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232998996990973, + "loss": 2.7513, + "theoretical_loss": 3.8872249966181442, + "tokens_seen": 534183936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232898696088265, + "loss": 2.9891, + "theoretical_loss": 3.887174217598461, + "tokens_seen": 534249472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232798395185557, + "loss": 2.7367, + "theoretical_loss": 3.8871234465513056, + "tokens_seen": 534315008 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042326980942828486, + "loss": 3.0058, + "theoretical_loss": 3.8870726834744485, + "tokens_seen": 534380544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232597793380141, + "loss": 2.7603, + "theoretical_loss": 3.8870219283656597, + "tokens_seen": 534446080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232497492477432, + "loss": 2.8634, + "theoretical_loss": 3.886971181222714, + "tokens_seen": 534511616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042323971915747246, + "loss": 2.8499, + "theoretical_loss": 3.886920442043383, + "tokens_seen": 534577152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232296890672016, + "loss": 2.9109, + "theoretical_loss": 3.886869710825442, + "tokens_seen": 534642688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232196589769308, + "loss": 2.884, + "theoretical_loss": 3.886818987566665, + "tokens_seen": 534708224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042320962888666, + "loss": 2.5804, + "theoretical_loss": 3.8867682722648293, + "tokens_seen": 534773760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231995987963892, + "loss": 2.7504, + "theoretical_loss": 3.886717564917711, + "tokens_seen": 534839296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042318956870611836, + "loss": 2.9655, + "theoretical_loss": 3.886666865523088, + "tokens_seen": 534904832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231795386158476, + "loss": 2.7419, + "theoretical_loss": 3.886616174078739, + "tokens_seen": 534970368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231695085255767, + "loss": 2.7549, + "theoretical_loss": 3.886565490582444, + "tokens_seen": 535035904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042315947843530596, + "loss": 2.9635, + "theoretical_loss": 3.8865148150319824, + "tokens_seen": 535101440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231494483450351, + "loss": 3.0665, + "theoretical_loss": 3.886464147425137, + "tokens_seen": 535166976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231394182547643, + "loss": 2.8279, + "theoretical_loss": 3.8864134877596896, + "tokens_seen": 535232512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231293881644935, + "loss": 2.7782, + "theoretical_loss": 3.886362836033423, + "tokens_seen": 535298048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231193580742227, + "loss": 2.3214, + "theoretical_loss": 3.8863121922441213, + "tokens_seen": 535363584 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042310932798395186, + "loss": 2.8828, + "theoretical_loss": 3.8862615563895693, + "tokens_seen": 535429120 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042309929789368105, + "loss": 2.8513, + "theoretical_loss": 3.886210928467553, + "tokens_seen": 535494656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230892678034102, + "loss": 2.9486, + "theoretical_loss": 3.8861603084758594, + "tokens_seen": 535560192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042307923771313946, + "loss": 2.845, + "theoretical_loss": 3.8861096964122757, + "tokens_seen": 535625728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230692076228686, + "loss": 2.7738, + "theoretical_loss": 3.886059092274591, + "tokens_seen": 535691264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 305312, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0122928619384766, + "objective/train/theoretical_loss": 3.886008496060594, + "objective/train/tokens_used": 556216800, + "theoretical_loss": 3.886008496060594, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230591775325978, + "loss": 2.9321, + "theoretical_loss": 3.886008496060594, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042304914744232695, + "loss": 2.8908, + "theoretical_loss": 3.8859579077680753, + "tokens_seen": 535822336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230391173520562, + "loss": 2.6939, + "theoretical_loss": 3.8859073273948255, + "tokens_seen": 535887872 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042302908726178537, + "loss": 2.8025, + "theoretical_loss": 3.8858567549386374, + "tokens_seen": 535953408 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042301905717151455, + "loss": 2.9458, + "theoretical_loss": 3.885806190397304, + "tokens_seen": 536018944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042300902708124373, + "loss": 2.6714, + "theoretical_loss": 3.8857556337686177, + "tokens_seen": 536084480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042299899699097297, + "loss": 2.8947, + "theoretical_loss": 3.8857050850503754, + "tokens_seen": 536150016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229889669007021, + "loss": 2.9313, + "theoretical_loss": 3.8856545442403707, + "tokens_seen": 536215552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042297893681043133, + "loss": 2.8646, + "theoretical_loss": 3.885604011336401, + "tokens_seen": 536281088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042296890672016045, + "loss": 2.829, + "theoretical_loss": 3.8855534863362635, + "tokens_seen": 536346624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229588766298897, + "loss": 2.9151, + "theoretical_loss": 3.8855029692377565, + "tokens_seen": 536412160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042294884653961887, + "loss": 3.0059, + "theoretical_loss": 3.885452460038679, + "tokens_seen": 536477696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042293881644934805, + "loss": 2.7475, + "theoretical_loss": 3.8854019587368307, + "tokens_seen": 536543232 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042292878635907723, + "loss": 2.8329, + "theoretical_loss": 3.8853514653300136, + "tokens_seen": 536608768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229187562688064, + "loss": 2.7283, + "theoretical_loss": 3.8853009798160274, + "tokens_seen": 536674304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229087261785356, + "loss": 2.659, + "theoretical_loss": 3.885250502192677, + "tokens_seen": 536739840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042289869608826483, + "loss": 2.8879, + "theoretical_loss": 3.8852000324577647, + "tokens_seen": 536805376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042288866599799396, + "loss": 2.8903, + "theoretical_loss": 3.8851495706090944, + "tokens_seen": 536870912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228786359077232, + "loss": 2.5484, + "theoretical_loss": 3.8850991166444726, + "tokens_seen": 536936448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228686058174523, + "loss": 2.8039, + "theoretical_loss": 3.885048670561705, + "tokens_seen": 537001984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042285857572718155, + "loss": 2.8749, + "theoretical_loss": 3.8849982323585985, + "tokens_seen": 537067520 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284854563691074, + "loss": 2.8962, + "theoretical_loss": 3.8849478020329604, + "tokens_seen": 537133056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228385155466399, + "loss": 2.9466, + "theoretical_loss": 3.8848973795826005, + "tokens_seen": 537198592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228284854563691, + "loss": 2.796, + "theoretical_loss": 3.884846965005328, + "tokens_seen": 537264128 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042281845536609833, + "loss": 2.6911, + "theoretical_loss": 3.8847965582989534, + "tokens_seen": 537329664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 306391, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.048434257507324, + "objective/train/theoretical_loss": 3.8847461594612884, + "objective/train/tokens_used": 557855200, + "theoretical_loss": 3.8847461594612884, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042280842527582746, + "loss": 2.8548, + "theoretical_loss": 3.8847461594612884, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227983951855567, + "loss": 2.8251, + "theoretical_loss": 3.884695768490145, + "tokens_seen": 537460736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227883650952858, + "loss": 2.8225, + "theoretical_loss": 3.8846453853833367, + "tokens_seen": 537526272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042277833500501506, + "loss": 2.9452, + "theoretical_loss": 3.8845950101386766, + "tokens_seen": 537591808 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042276830491474424, + "loss": 2.7989, + "theoretical_loss": 3.8845446427539807, + "tokens_seen": 537657344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227582748244734, + "loss": 2.6692, + "theoretical_loss": 3.884494283227064, + "tokens_seen": 537722880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227482447342026, + "loss": 2.9536, + "theoretical_loss": 3.884443931555744, + "tokens_seen": 537788416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227382146439318, + "loss": 3.0258, + "theoretical_loss": 3.884393587737838, + "tokens_seen": 537853952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042272818455366096, + "loss": 2.7838, + "theoretical_loss": 3.8843432517711634, + "tokens_seen": 537919488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227181544633902, + "loss": 2.7952, + "theoretical_loss": 3.8842929236535406, + "tokens_seen": 537985024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227081243731193, + "loss": 3.0073, + "theoretical_loss": 3.8842426033827895, + "tokens_seen": 538050560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042269809428284856, + "loss": 2.7916, + "theoretical_loss": 3.8841922909567312, + "tokens_seen": 538116096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226880641925777, + "loss": 2.8229, + "theoretical_loss": 3.8841419863731863, + "tokens_seen": 538181632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226780341023069, + "loss": 2.6835, + "theoretical_loss": 3.8840916896299795, + "tokens_seen": 538247168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042266800401203616, + "loss": 2.8953, + "theoretical_loss": 3.8840414007249335, + "tokens_seen": 538312704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226579739217653, + "loss": 2.7475, + "theoretical_loss": 3.8839911196558727, + "tokens_seen": 538378240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226479438314945, + "loss": 2.9781, + "theoretical_loss": 3.8839408464206224, + "tokens_seen": 538443776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226379137412237, + "loss": 3.0243, + "theoretical_loss": 3.8838905810170097, + "tokens_seen": 538509312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226278836509529, + "loss": 3.0065, + "theoretical_loss": 3.8838403234428602, + "tokens_seen": 538574848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042261785356068206, + "loss": 2.9446, + "theoretical_loss": 3.8837900736960034, + "tokens_seen": 538640384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042260782347041125, + "loss": 2.8828, + "theoretical_loss": 3.8837398317742666, + "tokens_seen": 538705920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225977933801404, + "loss": 2.9098, + "theoretical_loss": 3.883689597675481, + "tokens_seen": 538771456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042258776328986966, + "loss": 2.9336, + "theoretical_loss": 3.8836393713974764, + "tokens_seen": 538836992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225777331995988, + "loss": 3.1058, + "theoretical_loss": 3.883589152938084, + "tokens_seen": 538902528 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422567703109328, + "loss": 2.8419, + "theoretical_loss": 3.8835389422951367, + "tokens_seen": 538968064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 307598, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.633847236633301, + "objective/train/theoretical_loss": 3.883488739466467, + "objective/train/tokens_used": 559493600, + "theoretical_loss": 3.883488739466467, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042255767301905715, + "loss": 2.788, + "theoretical_loss": 3.883488739466467, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225476429287864, + "loss": 2.76, + "theoretical_loss": 3.8834385444499095, + "tokens_seen": 539099136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042253761283851557, + "loss": 2.8685, + "theoretical_loss": 3.883388357243299, + "tokens_seen": 539164672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252758274824475, + "loss": 2.7135, + "theoretical_loss": 3.883338177844471, + "tokens_seen": 539230208 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042251755265797393, + "loss": 2.9524, + "theoretical_loss": 3.883288006251262, + "tokens_seen": 539295744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042250752256770317, + "loss": 2.871, + "theoretical_loss": 3.88323784246151, + "tokens_seen": 539361280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224974924774323, + "loss": 2.7594, + "theoretical_loss": 3.883187686473053, + "tokens_seen": 539426816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248746238716153, + "loss": 2.8279, + "theoretical_loss": 3.8831375382837305, + "tokens_seen": 539492352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042247743229689065, + "loss": 2.7441, + "theoretical_loss": 3.883087397891382, + "tokens_seen": 539557888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224674022066199, + "loss": 3.0017, + "theoretical_loss": 3.883037265293849, + "tokens_seen": 539623424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042245737211634907, + "loss": 2.7708, + "theoretical_loss": 3.8829871404889724, + "tokens_seen": 539688960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042244734202607825, + "loss": 2.9023, + "theoretical_loss": 3.8829370234745957, + "tokens_seen": 539754496 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042243731193580743, + "loss": 2.7448, + "theoretical_loss": 3.8828869142485622, + "tokens_seen": 539820032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224272818455366, + "loss": 3.064, + "theoretical_loss": 3.8828368128087165, + "tokens_seen": 539885568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224172517552658, + "loss": 2.834, + "theoretical_loss": 3.882786719152903, + "tokens_seen": 539951104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042240722166499503, + "loss": 2.7914, + "theoretical_loss": 3.8827366332789683, + "tokens_seen": 540016640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042239719157472416, + "loss": 2.7571, + "theoretical_loss": 3.882686555184759, + "tokens_seen": 540082176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223871614844534, + "loss": 2.812, + "theoretical_loss": 3.882636484868124, + "tokens_seen": 540147712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223771313941825, + "loss": 3.0633, + "theoretical_loss": 3.882586422326911, + "tokens_seen": 540213248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042236710130391176, + "loss": 2.8002, + "theoretical_loss": 3.882536367558969, + "tokens_seen": 540278784 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042235707121364094, + "loss": 2.7051, + "theoretical_loss": 3.8824863205621494, + "tokens_seen": 540344320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223470411233701, + "loss": 2.7578, + "theoretical_loss": 3.882436281334303, + "tokens_seen": 540409856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223370110330993, + "loss": 2.8951, + "theoretical_loss": 3.8823862498732815, + "tokens_seen": 540475392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042232698094282853, + "loss": 2.7626, + "theoretical_loss": 3.882336226176939, + "tokens_seen": 540540928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042231695085255766, + "loss": 2.8586, + "theoretical_loss": 3.8822862102431275, + "tokens_seen": 540606464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 308194, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8666329383850098, + "objective/train/theoretical_loss": 3.882236202069703, + "objective/train/tokens_used": 561132000, + "theoretical_loss": 3.882236202069703, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223069207622869, + "loss": 2.8035, + "theoretical_loss": 3.882236202069703, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422296890672016, + "loss": 2.8564, + "theoretical_loss": 3.8821862016545206, + "tokens_seen": 540737536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042228686058174526, + "loss": 2.959, + "theoretical_loss": 3.8821362089954365, + "tokens_seen": 540803072 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042227683049147444, + "loss": 2.827, + "theoretical_loss": 3.8820862240903082, + "tokens_seen": 540868608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222668004012036, + "loss": 2.8683, + "theoretical_loss": 3.882036246936993, + "tokens_seen": 540934144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222567703109328, + "loss": 2.9829, + "theoretical_loss": 3.8819862775333505, + "tokens_seen": 540999680 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422246740220662, + "loss": 2.9701, + "theoretical_loss": 3.88193631587724, + "tokens_seen": 541065216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042223671013039116, + "loss": 2.8898, + "theoretical_loss": 3.8818863619665223, + "tokens_seen": 541130752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222266800401204, + "loss": 2.7672, + "theoretical_loss": 3.881836415799059, + "tokens_seen": 541196288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222166499498495, + "loss": 2.9484, + "theoretical_loss": 3.881786477372712, + "tokens_seen": 541261824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042220661985957876, + "loss": 2.8886, + "theoretical_loss": 3.881736546685345, + "tokens_seen": 541327360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221965897693079, + "loss": 3.0252, + "theoretical_loss": 3.8816866237348218, + "tokens_seen": 541392896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221865596790371, + "loss": 2.8781, + "theoretical_loss": 3.8816367085190064, + "tokens_seen": 541458432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221765295887663, + "loss": 2.9667, + "theoretical_loss": 3.8815868010357657, + "tokens_seen": 541523968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221664994984955, + "loss": 2.7153, + "theoretical_loss": 3.8815369012829657, + "tokens_seen": 541589504 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042215646940822467, + "loss": 2.5785, + "theoretical_loss": 3.881487009258473, + "tokens_seen": 541655040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221464393179539, + "loss": 2.8231, + "theoretical_loss": 3.881437124960157, + "tokens_seen": 541720576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042213640922768303, + "loss": 2.7915, + "theoretical_loss": 3.8813872483858867, + "tokens_seen": 541786112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042212637913741226, + "loss": 2.8312, + "theoretical_loss": 3.881337379533531, + "tokens_seen": 541851648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221163490471414, + "loss": 2.7949, + "theoretical_loss": 3.8812875184009616, + "tokens_seen": 541917184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221063189568706, + "loss": 2.8074, + "theoretical_loss": 3.88123766498605, + "tokens_seen": 541982720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220962888665998, + "loss": 2.7497, + "theoretical_loss": 3.881187819286668, + "tokens_seen": 542048256 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422086258776329, + "loss": 2.9072, + "theoretical_loss": 3.8811379813006894, + "tokens_seen": 542113792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042207622868605817, + "loss": 2.9012, + "theoretical_loss": 3.8810881510259883, + "tokens_seen": 542179328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042206619859578735, + "loss": 2.6692, + "theoretical_loss": 3.8810383284604395, + "tokens_seen": 542244864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 309537, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.619420289993286, + "objective/train/theoretical_loss": 3.8809885136019195, + "objective/train/tokens_used": 562770400, + "theoretical_loss": 3.8809885136019195, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042205616850551653, + "loss": 2.9674, + "theoretical_loss": 3.8809885136019195, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042204613841524577, + "loss": 2.7469, + "theoretical_loss": 3.880938706448304, + "tokens_seen": 542375936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220361083249749, + "loss": 2.8841, + "theoretical_loss": 3.8808889069974706, + "tokens_seen": 542441472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042202607823470413, + "loss": 2.8074, + "theoretical_loss": 3.880839115247298, + "tokens_seen": 542507008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220160481444333, + "loss": 2.7575, + "theoretical_loss": 3.880789331195665, + "tokens_seen": 542572544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220060180541625, + "loss": 2.8394, + "theoretical_loss": 3.8807395548404524, + "tokens_seen": 542638080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004219959879638917, + "loss": 2.7962, + "theoretical_loss": 3.8806897861795404, + "tokens_seen": 542703616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042198595787362085, + "loss": 2.9498, + "theoretical_loss": 3.880640025210811, + "tokens_seen": 542769152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042197592778335004, + "loss": 2.7269, + "theoretical_loss": 3.880590271932147, + "tokens_seen": 542834688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042196589769307927, + "loss": 2.8028, + "theoretical_loss": 3.8805405263414308, + "tokens_seen": 542900224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004219558676028084, + "loss": 2.8841, + "theoretical_loss": 3.8804907884365476, + "tokens_seen": 542965760 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042194583751253763, + "loss": 3.0371, + "theoretical_loss": 3.8804410582153817, + "tokens_seen": 543031296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042193580742226676, + "loss": 2.912, + "theoretical_loss": 3.8803913356758195, + "tokens_seen": 543096832 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421925777331996, + "loss": 2.7219, + "theoretical_loss": 3.8803416208157477, + "tokens_seen": 543162368 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042191574724172523, + "loss": 2.6821, + "theoretical_loss": 3.8802919136330534, + "tokens_seen": 543227904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042190571715145436, + "loss": 2.8521, + "theoretical_loss": 3.8802422141256256, + "tokens_seen": 543293440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218956870611836, + "loss": 2.7739, + "theoretical_loss": 3.8801925222913534, + "tokens_seen": 543358976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218856569709127, + "loss": 2.9423, + "theoretical_loss": 3.880142838128126, + "tokens_seen": 543424512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042187562688064196, + "loss": 2.7782, + "theoretical_loss": 3.880093161633836, + "tokens_seen": 543490048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042186559679037114, + "loss": 2.7181, + "theoretical_loss": 3.8800434928063736, + "tokens_seen": 543555584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218555667001003, + "loss": 2.8917, + "theoretical_loss": 3.879993831643632, + "tokens_seen": 543621120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218455366098295, + "loss": 3.0724, + "theoretical_loss": 3.879944178143505, + "tokens_seen": 543686656 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042183550651955873, + "loss": 2.8608, + "theoretical_loss": 3.879894532303886, + "tokens_seen": 543752192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042182547642928786, + "loss": 2.8151, + "theoretical_loss": 3.8798448941226704, + "tokens_seen": 543817728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218154463390171, + "loss": 2.7074, + "theoretical_loss": 3.879795263597754, + "tokens_seen": 543883264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 310176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.820920705795288, + "objective/train/theoretical_loss": 3.8797456407270343, + "objective/train/tokens_used": 564408800, + "theoretical_loss": 3.8797456407270343, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218054162487462, + "loss": 2.8936, + "theoretical_loss": 3.8797456407270343, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042179538615847546, + "loss": 2.8416, + "theoretical_loss": 3.879696025508408, + "tokens_seen": 544014336 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042178535606820464, + "loss": 2.9673, + "theoretical_loss": 3.8796464179397736, + "tokens_seen": 544079872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217753259779338, + "loss": 3.0778, + "theoretical_loss": 3.8795968180190306, + "tokens_seen": 544145408 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421765295887663, + "loss": 2.9471, + "theoretical_loss": 3.8795472257440786, + "tokens_seen": 544210944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217552657973922, + "loss": 2.7651, + "theoretical_loss": 3.8794976411128186, + "tokens_seen": 544276480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042174523570712136, + "loss": 2.761, + "theoretical_loss": 3.8794480641231535, + "tokens_seen": 544342016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217352056168506, + "loss": 3.0898, + "theoretical_loss": 3.879398494772984, + "tokens_seen": 544407552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217251755265797, + "loss": 2.7585, + "theoretical_loss": 3.879348933060215, + "tokens_seen": 544473088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042171514543630896, + "loss": 2.9004, + "theoretical_loss": 3.8792993789827497, + "tokens_seen": 544538624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004217051153460381, + "loss": 2.7262, + "theoretical_loss": 3.879249832538493, + "tokens_seen": 544604160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216950852557673, + "loss": 2.8847, + "theoretical_loss": 3.879200293725352, + "tokens_seen": 544669696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216850551654965, + "loss": 2.6465, + "theoretical_loss": 3.879150762541232, + "tokens_seen": 544735232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216750250752257, + "loss": 2.953, + "theoretical_loss": 3.8791012389840414, + "tokens_seen": 544800768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042166499498495487, + "loss": 2.6372, + "theoretical_loss": 3.879051723051688, + "tokens_seen": 544866304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216549648946841, + "loss": 2.8553, + "theoretical_loss": 3.8790022147420817, + "tokens_seen": 544931840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042164493480441323, + "loss": 2.861, + "theoretical_loss": 3.8789527140531312, + "tokens_seen": 544997376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042163490471414246, + "loss": 2.9117, + "theoretical_loss": 3.8789032209827488, + "tokens_seen": 545062912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216248746238716, + "loss": 2.7233, + "theoretical_loss": 3.8788537355288444, + "tokens_seen": 545128448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216148445336008, + "loss": 2.8419, + "theoretical_loss": 3.8788042576893322, + "tokens_seen": 545193984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042160481444333, + "loss": 2.7416, + "theoretical_loss": 3.8787547874621247, + "tokens_seen": 545259520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215947843530592, + "loss": 2.9282, + "theoretical_loss": 3.8787053248451357, + "tokens_seen": 545325056 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042158475426278837, + "loss": 2.8425, + "theoretical_loss": 3.8786558698362805, + "tokens_seen": 545390592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042157472417251755, + "loss": 2.9664, + "theoretical_loss": 3.8786064224334744, + "tokens_seen": 545456128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042156469408224673, + "loss": 2.6686, + "theoretical_loss": 3.8785569826346347, + "tokens_seen": 545521664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 311361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7344307899475098, + "objective/train/theoretical_loss": 3.8785075504376785, + "objective/train/tokens_used": 566047200, + "theoretical_loss": 3.8785075504376785, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042155466399197597, + "loss": 2.9008, + "theoretical_loss": 3.8785075504376785, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215446339017051, + "loss": 2.8395, + "theoretical_loss": 3.878458125840524, + "tokens_seen": 545652736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042153460381143433, + "loss": 2.852, + "theoretical_loss": 3.87840870884109, + "tokens_seen": 545718272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215245737211635, + "loss": 2.828, + "theoretical_loss": 3.878359299437296, + "tokens_seen": 545783808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215145436308927, + "loss": 2.9685, + "theoretical_loss": 3.878309897627064, + "tokens_seen": 545849344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215045135406219, + "loss": 2.9763, + "theoretical_loss": 3.878260503408314, + "tokens_seen": 545914880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042149448345035105, + "loss": 2.9497, + "theoretical_loss": 3.878211116778969, + "tokens_seen": 545980416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042148445336008024, + "loss": 2.9101, + "theoretical_loss": 3.878161737736952, + "tokens_seen": 546045952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042147442326980947, + "loss": 2.5154, + "theoretical_loss": 3.8781123662801873, + "tokens_seen": 546111488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214643931795386, + "loss": 2.8239, + "theoretical_loss": 3.878063002406599, + "tokens_seen": 546177024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042145436308926783, + "loss": 2.9453, + "theoretical_loss": 3.878013646114113, + "tokens_seen": 546242560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042144433299899696, + "loss": 2.692, + "theoretical_loss": 3.8779642974006556, + "tokens_seen": 546308096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214343029087262, + "loss": 2.7597, + "theoretical_loss": 3.877914956264154, + "tokens_seen": 546373632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214242728184554, + "loss": 2.8578, + "theoretical_loss": 3.8778656227025365, + "tokens_seen": 546439168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042141424272818456, + "loss": 2.9577, + "theoretical_loss": 3.877816296713732, + "tokens_seen": 546504704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042140421263791374, + "loss": 2.9289, + "theoretical_loss": 3.8777669782956696, + "tokens_seen": 546570240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213941825476429, + "loss": 2.9022, + "theoretical_loss": 3.8777176674462797, + "tokens_seen": 546635776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213841524573721, + "loss": 2.8726, + "theoretical_loss": 3.877668364163494, + "tokens_seen": 546701312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042137412236710134, + "loss": 2.6827, + "theoretical_loss": 3.8776190684452447, + "tokens_seen": 546766848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042136409227683046, + "loss": 2.8975, + "theoretical_loss": 3.8775697802894644, + "tokens_seen": 546832384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213540621865597, + "loss": 2.8235, + "theoretical_loss": 3.8775204996940866, + "tokens_seen": 546897920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213440320962889, + "loss": 2.7518, + "theoretical_loss": 3.877471226657047, + "tokens_seen": 546963456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042133400200601806, + "loss": 2.9066, + "theoretical_loss": 3.877421961176279, + "tokens_seen": 547028992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042132397191574724, + "loss": 3.0313, + "theoretical_loss": 3.87737270324972, + "tokens_seen": 547094528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213139418254764, + "loss": 3.0107, + "theoretical_loss": 3.8773234528753076, + "tokens_seen": 547160064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 312508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.020892381668091, + "objective/train/theoretical_loss": 3.8772742100509774, + "objective/train/tokens_used": 567685600, + "theoretical_loss": 3.8772742100509774, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213039117352056, + "loss": 2.8083, + "theoretical_loss": 3.8772742100509774, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042129388164493484, + "loss": 2.5294, + "theoretical_loss": 3.8772249747746703, + "tokens_seen": 547291136 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042128385155466397, + "loss": 3.0742, + "theoretical_loss": 3.8771757470443244, + "tokens_seen": 547356672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212738214643932, + "loss": 2.6558, + "theoretical_loss": 3.87712652685788, + "tokens_seen": 547422208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042126379137412233, + "loss": 3.0363, + "theoretical_loss": 3.8770773142132784, + "tokens_seen": 547487744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042125376128385156, + "loss": 2.8451, + "theoretical_loss": 3.8770281091084615, + "tokens_seen": 547553280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042124373119358075, + "loss": 2.7166, + "theoretical_loss": 3.8769789115413715, + "tokens_seen": 547618816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212337011033099, + "loss": 2.7842, + "theoretical_loss": 3.8769297215099523, + "tokens_seen": 547684352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212236710130391, + "loss": 3.1148, + "theoretical_loss": 3.8768805390121477, + "tokens_seen": 547749888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212136409227683, + "loss": 2.8194, + "theoretical_loss": 3.876831364045903, + "tokens_seen": 547815424 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042120361083249747, + "loss": 2.7916, + "theoretical_loss": 3.876782196609164, + "tokens_seen": 547880960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211935807422267, + "loss": 2.4156, + "theoretical_loss": 3.8767330366998776, + "tokens_seen": 547946496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042118355065195583, + "loss": 2.9464, + "theoretical_loss": 3.8766838843159905, + "tokens_seen": 548012032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042117352056168507, + "loss": 2.8205, + "theoretical_loss": 3.8766347394554517, + "tokens_seen": 548077568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211634904714143, + "loss": 2.9376, + "theoretical_loss": 3.87658560211621, + "tokens_seen": 548143104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042115346038114343, + "loss": 2.8582, + "theoretical_loss": 3.8765364722962152, + "tokens_seen": 548208640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042114343029087266, + "loss": 2.9846, + "theoretical_loss": 3.8764873499934183, + "tokens_seen": 548274176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211334002006018, + "loss": 2.7357, + "theoretical_loss": 3.8764382352057707, + "tokens_seen": 548339712 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042112337011033103, + "loss": 2.911, + "theoretical_loss": 3.8763891279312244, + "tokens_seen": 548405248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211133400200602, + "loss": 2.8353, + "theoretical_loss": 3.876340028167732, + "tokens_seen": 548470784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211033099297894, + "loss": 2.8009, + "theoretical_loss": 3.876290935913249, + "tokens_seen": 548536320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042109327983951857, + "loss": 2.7628, + "theoretical_loss": 3.876241851165729, + "tokens_seen": 548601856 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042108324974924775, + "loss": 2.74, + "theoretical_loss": 3.876192773923127, + "tokens_seen": 548667392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042107321965897693, + "loss": 3.0434, + "theoretical_loss": 3.8761437041834004, + "tokens_seen": 548732928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042106318956870617, + "loss": 2.8916, + "theoretical_loss": 3.8760946419445057, + "tokens_seen": 548798464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 312943, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1393747329711914, + "objective/train/theoretical_loss": 3.876045587204401, + "objective/train/tokens_used": 569324000, + "theoretical_loss": 3.876045587204401, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210531594784353, + "loss": 2.7268, + "theoretical_loss": 3.876045587204401, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042104312938816453, + "loss": 2.7706, + "theoretical_loss": 3.875996539961045, + "tokens_seen": 548929536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210330992978937, + "loss": 2.8024, + "theoretical_loss": 3.875947500212397, + "tokens_seen": 548995072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210230692076229, + "loss": 2.8611, + "theoretical_loss": 3.8758984679564175, + "tokens_seen": 549060608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210130391173521, + "loss": 2.5877, + "theoretical_loss": 3.875849443191067, + "tokens_seen": 549126144 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042100300902708125, + "loss": 2.7126, + "theoretical_loss": 3.875800425914308, + "tokens_seen": 549191680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042099297893681044, + "loss": 2.8382, + "theoretical_loss": 3.8757514161241033, + "tokens_seen": 549257216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042098294884653967, + "loss": 2.8309, + "theoretical_loss": 3.8757024138184164, + "tokens_seen": 549322752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209729187562688, + "loss": 2.7394, + "theoretical_loss": 3.8756534189952108, + "tokens_seen": 549388288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042096288866599803, + "loss": 2.8002, + "theoretical_loss": 3.8756044316524525, + "tokens_seen": 549453824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042095285857572716, + "loss": 2.8171, + "theoretical_loss": 3.875555451788107, + "tokens_seen": 549519360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209428284854564, + "loss": 2.8046, + "theoretical_loss": 3.8755064794001406, + "tokens_seen": 549584896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209327983951856, + "loss": 2.8639, + "theoretical_loss": 3.875457514486521, + "tokens_seen": 549650432 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042092276830491476, + "loss": 2.8338, + "theoretical_loss": 3.8754085570452164, + "tokens_seen": 549715968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042091273821464394, + "loss": 2.9081, + "theoretical_loss": 3.8753596070741967, + "tokens_seen": 549781504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209027081243731, + "loss": 2.5443, + "theoretical_loss": 3.8753106645714306, + "tokens_seen": 549847040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208926780341023, + "loss": 2.587, + "theoretical_loss": 3.8752617295348895, + "tokens_seen": 549912576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042088264794383154, + "loss": 2.7818, + "theoretical_loss": 3.8752128019625443, + "tokens_seen": 549978112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042087261785356066, + "loss": 2.8385, + "theoretical_loss": 3.8751638818523677, + "tokens_seen": 550043648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208625877632899, + "loss": 2.7451, + "theoretical_loss": 3.875114969202332, + "tokens_seen": 550109184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208525576730191, + "loss": 2.7683, + "theoretical_loss": 3.8750660640104124, + "tokens_seen": 550174720 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042084252758274826, + "loss": 2.8129, + "theoretical_loss": 3.875017166274582, + "tokens_seen": 550240256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042083249749247744, + "loss": 3.0056, + "theoretical_loss": 3.8749682759928166, + "tokens_seen": 550305792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208224674022066, + "loss": 2.8207, + "theoretical_loss": 3.8749193931630925, + "tokens_seen": 550371328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208124373119358, + "loss": 2.9711, + "theoretical_loss": 3.8748705177833873, + "tokens_seen": 550436864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 314102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.005920886993408, + "objective/train/theoretical_loss": 3.874821649851678, + "objective/train/tokens_used": 570962400, + "theoretical_loss": 3.874821649851678, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042080240722166504, + "loss": 3.0135, + "theoretical_loss": 3.874821649851678, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042079237713139417, + "loss": 2.6148, + "theoretical_loss": 3.8747727893659434, + "tokens_seen": 550567936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207823470411234, + "loss": 2.7024, + "theoretical_loss": 3.8747239363241626, + "tokens_seen": 550633472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042077231695085253, + "loss": 2.698, + "theoretical_loss": 3.8746750907243164, + "tokens_seen": 550699008 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042076228686058176, + "loss": 3.0277, + "theoretical_loss": 3.8746262525643846, + "tokens_seen": 550764544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042075225677031095, + "loss": 2.8225, + "theoretical_loss": 3.8745774218423508, + "tokens_seen": 550830080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207422266800401, + "loss": 2.9363, + "theoretical_loss": 3.874528598556195, + "tokens_seen": 550895616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207321965897693, + "loss": 2.7584, + "theoretical_loss": 3.8744797827039026, + "tokens_seen": 550961152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207221664994985, + "loss": 2.7688, + "theoretical_loss": 3.874430974283456, + "tokens_seen": 551026688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042071213640922767, + "loss": 2.9367, + "theoretical_loss": 3.874382173292842, + "tokens_seen": 551092224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207021063189569, + "loss": 2.7892, + "theoretical_loss": 3.874333379730044, + "tokens_seen": 551157760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042069207622868603, + "loss": 2.8285, + "theoretical_loss": 3.8742845935930506, + "tokens_seen": 551223296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042068204613841527, + "loss": 2.6297, + "theoretical_loss": 3.8742358148798473, + "tokens_seen": 551288832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042067201604814445, + "loss": 2.8855, + "theoretical_loss": 3.8741870435884227, + "tokens_seen": 551354368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042066198595787363, + "loss": 2.7368, + "theoretical_loss": 3.874138279716766, + "tokens_seen": 551419904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206519558676028, + "loss": 2.7036, + "theoretical_loss": 3.8740895232628665, + "tokens_seen": 551485440 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420641925777332, + "loss": 2.9885, + "theoretical_loss": 3.874040774224714, + "tokens_seen": 551550976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206318956870612, + "loss": 2.836, + "theoretical_loss": 3.8739920326003006, + "tokens_seen": 551616512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206218655967904, + "loss": 2.8656, + "theoretical_loss": 3.8739432983876174, + "tokens_seen": 551682048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042061183550651954, + "loss": 2.766, + "theoretical_loss": 3.8738945715846578, + "tokens_seen": 551747584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042060180541624877, + "loss": 2.8527, + "theoretical_loss": 3.8738458521894144, + "tokens_seen": 551813120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205917753259779, + "loss": 2.8719, + "theoretical_loss": 3.8737971401998825, + "tokens_seen": 551878656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042058174523570713, + "loss": 2.7869, + "theoretical_loss": 3.8737484356140564, + "tokens_seen": 551944192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205717151454363, + "loss": 2.759, + "theoretical_loss": 3.873699738429932, + "tokens_seen": 552009728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205616850551655, + "loss": 2.8348, + "theoretical_loss": 3.873651048645506, + "tokens_seen": 552075264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 314596, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6391680240631104, + "objective/train/theoretical_loss": 3.873602366258776, + "objective/train/tokens_used": 572600800, + "theoretical_loss": 3.873602366258776, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205516549648947, + "loss": 2.7018, + "theoretical_loss": 3.873602366258776, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205416248746239, + "loss": 3.0718, + "theoretical_loss": 3.87355369126774, + "tokens_seen": 552206336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042053159478435304, + "loss": 2.9741, + "theoretical_loss": 3.873505023670397, + "tokens_seen": 552271872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205215646940823, + "loss": 2.6878, + "theoretical_loss": 3.8734563634647463, + "tokens_seen": 552337408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205115346038114, + "loss": 2.7524, + "theoretical_loss": 3.8734077106487894, + "tokens_seen": 552402944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042050150451354064, + "loss": 2.8489, + "theoretical_loss": 3.8733590652205265, + "tokens_seen": 552468480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204914744232698, + "loss": 2.8195, + "theoretical_loss": 3.8733104271779606, + "tokens_seen": 552534016 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420481444332999, + "loss": 2.7318, + "theoretical_loss": 3.8732617965190936, + "tokens_seen": 552599552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204714142427282, + "loss": 2.7373, + "theoretical_loss": 3.873213173241929, + "tokens_seen": 552665088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042046138415245736, + "loss": 2.7681, + "theoretical_loss": 3.8731645573444724, + "tokens_seen": 552730624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042045135406218654, + "loss": 2.801, + "theoretical_loss": 3.8731159488247284, + "tokens_seen": 552796160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204413239719158, + "loss": 2.7685, + "theoretical_loss": 3.8730673476807023, + "tokens_seen": 552861696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204312938816449, + "loss": 2.8712, + "theoretical_loss": 3.8730187539104017, + "tokens_seen": 552927232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042042126379137414, + "loss": 2.7277, + "theoretical_loss": 3.8729701675118338, + "tokens_seen": 552992768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204112337011033, + "loss": 2.8187, + "theoretical_loss": 3.8729215884830066, + "tokens_seen": 553058304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204012036108325, + "loss": 2.6754, + "theoretical_loss": 3.8728730168219294, + "tokens_seen": 553123840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042039117352056174, + "loss": 2.6005, + "theoretical_loss": 3.872824452526612, + "tokens_seen": 553189376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042038114343029086, + "loss": 2.6217, + "theoretical_loss": 3.8727758955950646, + "tokens_seen": 553254912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203711133400201, + "loss": 2.9321, + "theoretical_loss": 3.8727273460252993, + "tokens_seen": 553320448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203610832497493, + "loss": 2.6787, + "theoretical_loss": 3.8726788038153277, + "tokens_seen": 553385984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042035105315947846, + "loss": 2.8041, + "theoretical_loss": 3.872630268963163, + "tokens_seen": 553451520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042034102306920764, + "loss": 2.811, + "theoretical_loss": 3.872581741466818, + "tokens_seen": 553517056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203309929789368, + "loss": 2.6364, + "theoretical_loss": 3.8725332213243084, + "tokens_seen": 553582592 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420320962888666, + "loss": 2.6177, + "theoretical_loss": 3.8724847085336487, + "tokens_seen": 553648128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042031093279839524, + "loss": 2.7531, + "theoretical_loss": 3.872436203092855, + "tokens_seen": 553713664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 315339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5412378311157227, + "objective/train/theoretical_loss": 3.8723877049999444, + "objective/train/tokens_used": 574239200, + "theoretical_loss": 3.8723877049999444, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042030090270812437, + "loss": 2.4705, + "theoretical_loss": 3.8723877049999444, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202908726178536, + "loss": 2.6838, + "theoretical_loss": 3.872339214252934, + "tokens_seen": 553844736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042028084252758273, + "loss": 2.6182, + "theoretical_loss": 3.8722907308498415, + "tokens_seen": 553910272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042027081243731196, + "loss": 2.8419, + "theoretical_loss": 3.8722422547886874, + "tokens_seen": 553975808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042026078234704115, + "loss": 2.7933, + "theoretical_loss": 3.8721937860674904, + "tokens_seen": 554041344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202507522567703, + "loss": 2.9367, + "theoretical_loss": 3.872145324684272, + "tokens_seen": 554106880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202407221664995, + "loss": 2.5143, + "theoretical_loss": 3.8720968706370527, + "tokens_seen": 554172416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202306920762287, + "loss": 2.8308, + "theoretical_loss": 3.8720484239238555, + "tokens_seen": 554237952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042022066198595787, + "loss": 2.707, + "theoretical_loss": 3.8719999845427022, + "tokens_seen": 554303488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202106318956871, + "loss": 2.4634, + "theoretical_loss": 3.871951552491617, + "tokens_seen": 554369024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042020060180541623, + "loss": 2.6944, + "theoretical_loss": 3.871903127768625, + "tokens_seen": 554434560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042019057171514547, + "loss": 2.8056, + "theoretical_loss": 3.8718547103717507, + "tokens_seen": 554500096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042018054162487465, + "loss": 2.7418, + "theoretical_loss": 3.87180630029902, + "tokens_seen": 554565632 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042017051153460383, + "loss": 2.9632, + "theoretical_loss": 3.87175789754846, + "tokens_seen": 554631168 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420160481444333, + "loss": 2.5859, + "theoretical_loss": 3.8717095021180983, + "tokens_seen": 554696704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201504513540622, + "loss": 2.8359, + "theoretical_loss": 3.8716611140059625, + "tokens_seen": 554762240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201404212637914, + "loss": 2.6759, + "theoretical_loss": 3.871612733210082, + "tokens_seen": 554827776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201303911735206, + "loss": 2.7389, + "theoretical_loss": 3.8715643597284863, + "tokens_seen": 554893312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042012036108324974, + "loss": 2.7195, + "theoretical_loss": 3.871515993559207, + "tokens_seen": 554958848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042011033099297897, + "loss": 2.814, + "theoretical_loss": 3.871467634700274, + "tokens_seen": 555024384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201003009027081, + "loss": 2.5207, + "theoretical_loss": 3.8714192831497205, + "tokens_seen": 555089920 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042009027081243733, + "loss": 2.6039, + "theoretical_loss": 3.871370938905579, + "tokens_seen": 555155456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200802407221665, + "loss": 3.1536, + "theoretical_loss": 3.8713226019658826, + "tokens_seen": 555220992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200702106318957, + "loss": 2.6002, + "theoretical_loss": 3.871274272328666, + "tokens_seen": 555286528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200601805416249, + "loss": 2.8405, + "theoretical_loss": 3.871225949991965, + "tokens_seen": 555352064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 316316, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.795095682144165, + "objective/train/theoretical_loss": 3.8711776349538143, + "objective/train/tokens_used": 575877600, + "theoretical_loss": 3.8711776349538143, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200501504513541, + "loss": 2.7872, + "theoretical_loss": 3.8711776349538143, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042004012036108324, + "loss": 2.8702, + "theoretical_loss": 3.8711293272122513, + "tokens_seen": 555483136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200300902708125, + "loss": 2.9276, + "theoretical_loss": 3.871081026765313, + "tokens_seen": 555548672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200200601805416, + "loss": 2.7231, + "theoretical_loss": 3.871032733611038, + "tokens_seen": 555614208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042001003009027084, + "loss": 2.8501, + "theoretical_loss": 3.870984447747465, + "tokens_seen": 555679744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042, + "loss": 2.8278, + "theoretical_loss": 3.8709361691726336, + "tokens_seen": 555745280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199899699097292, + "loss": 2.8936, + "theoretical_loss": 3.870887897884584, + "tokens_seen": 555810816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199799398194584, + "loss": 2.6572, + "theoretical_loss": 3.870839633881358, + "tokens_seen": 555876352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041996990972918756, + "loss": 2.9405, + "theoretical_loss": 3.8707913771609976, + "tokens_seen": 555941888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041995987963891674, + "loss": 2.4867, + "theoretical_loss": 3.8707431277215454, + "tokens_seen": 556007424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419949849548646, + "loss": 2.8431, + "theoretical_loss": 3.8706948855610435, + "tokens_seen": 556072960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199398194583751, + "loss": 2.947, + "theoretical_loss": 3.8706466506775383, + "tokens_seen": 556138496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041992978936810434, + "loss": 3.0076, + "theoretical_loss": 3.8705984230690733, + "tokens_seen": 556204032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041991975927783347, + "loss": 2.8488, + "theoretical_loss": 3.870550202733695, + "tokens_seen": 556269568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199097291875627, + "loss": 2.6245, + "theoretical_loss": 3.8705019896694495, + "tokens_seen": 556335104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198996990972919, + "loss": 2.8031, + "theoretical_loss": 3.8704537838743835, + "tokens_seen": 556400640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041988966900702106, + "loss": 2.6974, + "theoretical_loss": 3.870405585346546, + "tokens_seen": 556466176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041987963891675025, + "loss": 2.6182, + "theoretical_loss": 3.8703573940839857, + "tokens_seen": 556531712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198696088264795, + "loss": 2.6669, + "theoretical_loss": 3.8703092100847516, + "tokens_seen": 556597248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198595787362086, + "loss": 2.8824, + "theoretical_loss": 3.8702610333468943, + "tokens_seen": 556662784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041984954864593784, + "loss": 2.8379, + "theoretical_loss": 3.8702128638684643, + "tokens_seen": 556728320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041983951855566697, + "loss": 2.602, + "theoretical_loss": 3.870164701647514, + "tokens_seen": 556793856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198294884653962, + "loss": 2.84, + "theoretical_loss": 3.8701165466820955, + "tokens_seen": 556859392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198194583751254, + "loss": 2.7329, + "theoretical_loss": 3.870068398970262, + "tokens_seen": 556924928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041980942828485457, + "loss": 2.7621, + "theoretical_loss": 3.870020258510068, + "tokens_seen": 556990464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 317467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9780025482177734, + "objective/train/theoretical_loss": 3.869972125299568, + "objective/train/tokens_used": 577516000, + "theoretical_loss": 3.869972125299568, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041979939819458375, + "loss": 2.8299, + "theoretical_loss": 3.869972125299568, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041978936810431293, + "loss": 2.7222, + "theoretical_loss": 3.8699239993368177, + "tokens_seen": 557121536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197793380140421, + "loss": 2.6026, + "theoretical_loss": 3.869875880619873, + "tokens_seen": 557187072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041976930792377135, + "loss": 2.8492, + "theoretical_loss": 3.869827769146791, + "tokens_seen": 557252608 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041975927783350047, + "loss": 2.6935, + "theoretical_loss": 3.8697796649156295, + "tokens_seen": 557318144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197492477432297, + "loss": 2.5823, + "theoretical_loss": 3.8697315679244477, + "tokens_seen": 557383680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041973921765295883, + "loss": 2.7691, + "theoretical_loss": 3.8696834781713036, + "tokens_seen": 557449216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041972918756268807, + "loss": 2.6545, + "theoretical_loss": 3.8696353956542584, + "tokens_seen": 557514752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041971915747241725, + "loss": 2.6819, + "theoretical_loss": 3.869587320371372, + "tokens_seen": 557580288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041970912738214643, + "loss": 2.7852, + "theoretical_loss": 3.869539252320707, + "tokens_seen": 557645824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196990972918756, + "loss": 2.6624, + "theoretical_loss": 3.8694911915003245, + "tokens_seen": 557711360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041968906720160485, + "loss": 2.8304, + "theoretical_loss": 3.869443137908288, + "tokens_seen": 557776896 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419679037111334, + "loss": 2.9432, + "theoretical_loss": 3.869395091542661, + "tokens_seen": 557842432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196690070210632, + "loss": 2.6705, + "theoretical_loss": 3.8693470524015083, + "tokens_seen": 557907968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196589769307924, + "loss": 2.9572, + "theoretical_loss": 3.869299020482895, + "tokens_seen": 557973504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196489468405216, + "loss": 2.5857, + "theoretical_loss": 3.8692509957848875, + "tokens_seen": 558039040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196389167502508, + "loss": 2.8012, + "theoretical_loss": 3.8692029783055517, + "tokens_seen": 558104576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041962888665997994, + "loss": 2.8241, + "theoretical_loss": 3.869154968042956, + "tokens_seen": 558170112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041961885656970917, + "loss": 2.8948, + "theoretical_loss": 3.8691069649951677, + "tokens_seen": 558235648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196088264794383, + "loss": 2.827, + "theoretical_loss": 3.869058969160257, + "tokens_seen": 558301184 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041959879638916753, + "loss": 2.8092, + "theoretical_loss": 3.869010980536292, + "tokens_seen": 558366720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195887662988967, + "loss": 2.8753, + "theoretical_loss": 3.868962999121344, + "tokens_seen": 558432256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195787362086259, + "loss": 2.8558, + "theoretical_loss": 3.868915024913485, + "tokens_seen": 558497792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195687061183551, + "loss": 2.8519, + "theoretical_loss": 3.8688670579107853, + "tokens_seen": 558563328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195586760280843, + "loss": 2.9895, + "theoretical_loss": 3.8688190981113193, + "tokens_seen": 558628864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 318032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9913525581359863, + "objective/train/theoretical_loss": 3.868771145513159, + "objective/train/tokens_used": 579154400, + "theoretical_loss": 3.868771145513159, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041954864593781344, + "loss": 2.6942, + "theoretical_loss": 3.868771145513159, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195386158475427, + "loss": 2.7493, + "theoretical_loss": 3.8687232001143794, + "tokens_seen": 558759936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195285857572718, + "loss": 2.9329, + "theoretical_loss": 3.8686752619130553, + "tokens_seen": 558825472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041951855566700104, + "loss": 2.8755, + "theoretical_loss": 3.8686273309072616, + "tokens_seen": 558891008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195085255767302, + "loss": 2.7809, + "theoretical_loss": 3.8685794070950754, + "tokens_seen": 558956544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194984954864594, + "loss": 3.0322, + "theoretical_loss": 3.8685314904745742, + "tokens_seen": 559022080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194884653961886, + "loss": 2.6098, + "theoretical_loss": 3.8684835810438347, + "tokens_seen": 559087616 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041947843530591776, + "loss": 2.7776, + "theoretical_loss": 3.868435678800937, + "tokens_seen": 559153152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041946840521564694, + "loss": 2.7414, + "theoretical_loss": 3.868387783743959, + "tokens_seen": 559218688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194583751253762, + "loss": 2.883, + "theoretical_loss": 3.8683398958709816, + "tokens_seen": 559284224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194483450351053, + "loss": 2.6664, + "theoretical_loss": 3.868292015180085, + "tokens_seen": 559349760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943831494483454, + "loss": 2.7937, + "theoretical_loss": 3.8682441416693516, + "tokens_seen": 559415296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041942828485456367, + "loss": 2.8014, + "theoretical_loss": 3.868196275336863, + "tokens_seen": 559480832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194182547642929, + "loss": 2.8912, + "theoretical_loss": 3.8681484161807025, + "tokens_seen": 559546368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194082246740221, + "loss": 2.8199, + "theoretical_loss": 3.8681005641989543, + "tokens_seen": 559611904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041939819458375126, + "loss": 2.792, + "theoretical_loss": 3.8680527193897016, + "tokens_seen": 559677440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041938816449348045, + "loss": 2.8252, + "theoretical_loss": 3.8680048817510313, + "tokens_seen": 559742976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193781344032097, + "loss": 2.9691, + "theoretical_loss": 3.867957051281028, + "tokens_seen": 559808512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193681043129388, + "loss": 2.7795, + "theoretical_loss": 3.867909227977779, + "tokens_seen": 559874048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041935807422266804, + "loss": 2.7819, + "theoretical_loss": 3.8678614118393724, + "tokens_seen": 559939584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041934804413239717, + "loss": 2.6689, + "theoretical_loss": 3.8678136028638948, + "tokens_seen": 560005120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193380140421264, + "loss": 2.8671, + "theoretical_loss": 3.867765801049437, + "tokens_seen": 560070656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193279839518556, + "loss": 2.8617, + "theoretical_loss": 3.8677180063940866, + "tokens_seen": 560136192 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041931795386158477, + "loss": 2.8101, + "theoretical_loss": 3.8676702188959355, + "tokens_seen": 560201728 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041930792377131395, + "loss": 2.8684, + "theoretical_loss": 3.8676224385530746, + "tokens_seen": 560267264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 319260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.651021718978882, + "objective/train/theoretical_loss": 3.867574665363595, + "objective/train/tokens_used": 580792800, + "theoretical_loss": 3.867574665363595, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041929789368104313, + "loss": 2.8879, + "theoretical_loss": 3.867574665363595, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192878635907723, + "loss": 2.8662, + "theoretical_loss": 3.8675268993255902, + "tokens_seen": 560398336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927783350050155, + "loss": 2.8046, + "theoretical_loss": 3.867479140437153, + "tokens_seen": 560463872 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041926780341023067, + "loss": 2.7195, + "theoretical_loss": 3.867431388696377, + "tokens_seen": 560529408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192577733199599, + "loss": 2.6808, + "theoretical_loss": 3.867383644101358, + "tokens_seen": 560594944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041924774322968904, + "loss": 2.8332, + "theoretical_loss": 3.8673359066501902, + "tokens_seen": 560660480 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041923771313941827, + "loss": 2.884, + "theoretical_loss": 3.8672881763409714, + "tokens_seen": 560726016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041922768304914745, + "loss": 2.8302, + "theoretical_loss": 3.8672404531717977, + "tokens_seen": 560791552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041921765295887663, + "loss": 2.9092, + "theoretical_loss": 3.8671927371407664, + "tokens_seen": 560857088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192076228686058, + "loss": 2.8494, + "theoretical_loss": 3.8671450282459765, + "tokens_seen": 560922624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041919759277833505, + "loss": 2.8229, + "theoretical_loss": 3.867097326485527, + "tokens_seen": 560988160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191875626880642, + "loss": 2.8452, + "theoretical_loss": 3.867049631857518, + "tokens_seen": 561053696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191775325977934, + "loss": 2.8199, + "theoretical_loss": 3.86700194436005, + "tokens_seen": 561119232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041916750250752254, + "loss": 2.9338, + "theoretical_loss": 3.8669542639912238, + "tokens_seen": 561184768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191574724172518, + "loss": 3.0957, + "theoretical_loss": 3.866906590749142, + "tokens_seen": 561250304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041914744232698095, + "loss": 2.7999, + "theoretical_loss": 3.8668589246319067, + "tokens_seen": 561315840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041913741223671014, + "loss": 2.9401, + "theoretical_loss": 3.8668112656376223, + "tokens_seen": 561381376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191273821464393, + "loss": 2.7745, + "theoretical_loss": 3.8667636137643937, + "tokens_seen": 561446912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191173520561685, + "loss": 2.9357, + "theoretical_loss": 3.8667159690103237, + "tokens_seen": 561512448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191073219658977, + "loss": 2.6896, + "theoretical_loss": 3.8666683313735195, + "tokens_seen": 561577984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190972918756269, + "loss": 2.7651, + "theoretical_loss": 3.8666207008520868, + "tokens_seen": 561643520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041908726178535604, + "loss": 2.8477, + "theoretical_loss": 3.8665730774441336, + "tokens_seen": 561709056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190772316950853, + "loss": 3.0061, + "theoretical_loss": 3.866525461147767, + "tokens_seen": 561774592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190672016048144, + "loss": 2.7424, + "theoretical_loss": 3.866477851961096, + "tokens_seen": 561840128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041905717151454364, + "loss": 2.9924, + "theoretical_loss": 3.8664302498822294, + "tokens_seen": 561905664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 319818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6618897914886475, + "objective/train/theoretical_loss": 3.8663826549092777, + "objective/train/tokens_used": 582431200, + "theoretical_loss": 3.8663826549092777, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190471414242728, + "loss": 3.0017, + "theoretical_loss": 3.8663826549092777, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419037111334002, + "loss": 2.7453, + "theoretical_loss": 3.8663350670403513, + "tokens_seen": 562036736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190270812437312, + "loss": 2.7955, + "theoretical_loss": 3.8662874862735626, + "tokens_seen": 562102272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190170511534604, + "loss": 2.7434, + "theoretical_loss": 3.8662399126070226, + "tokens_seen": 562167808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041900702106318954, + "loss": 2.8103, + "theoretical_loss": 3.866192346038845, + "tokens_seen": 562233344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189969909729188, + "loss": 2.7913, + "theoretical_loss": 3.866144786567143, + "tokens_seen": 562298880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189869608826479, + "loss": 2.6216, + "theoretical_loss": 3.8660972341900313, + "tokens_seen": 562364416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041897693079237714, + "loss": 2.7139, + "theoretical_loss": 3.8660496889056244, + "tokens_seen": 562429952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189669007021063, + "loss": 3.0612, + "theoretical_loss": 3.8660021507120392, + "tokens_seen": 562495488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189568706118355, + "loss": 2.7447, + "theoretical_loss": 3.8659546196073906, + "tokens_seen": 562561024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189468405215647, + "loss": 2.624, + "theoretical_loss": 3.8659070955897974, + "tokens_seen": 562626560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041893681043129387, + "loss": 2.5809, + "theoretical_loss": 3.865859578657377, + "tokens_seen": 562692096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041892678034102305, + "loss": 2.6602, + "theoretical_loss": 3.865812068808248, + "tokens_seen": 562757632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189167502507523, + "loss": 2.7531, + "theoretical_loss": 3.86576456604053, + "tokens_seen": 562823168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041890672016048146, + "loss": 2.8401, + "theoretical_loss": 3.8657170703523427, + "tokens_seen": 562888704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041889669007021065, + "loss": 2.7188, + "theoretical_loss": 3.865669581741807, + "tokens_seen": 562954240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188866599799399, + "loss": 2.7441, + "theoretical_loss": 3.865622100207045, + "tokens_seen": 563019776 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418876629889669, + "loss": 2.7755, + "theoretical_loss": 3.8655746257461785, + "tokens_seen": 563085312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041886659979939824, + "loss": 2.8061, + "theoretical_loss": 3.8655271583573305, + "tokens_seen": 563150848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041885656970912737, + "loss": 2.8223, + "theoretical_loss": 3.8654796980386252, + "tokens_seen": 563216384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188465396188566, + "loss": 2.7483, + "theoretical_loss": 3.8654322447881864, + "tokens_seen": 563281920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188365095285858, + "loss": 3.0129, + "theoretical_loss": 3.8653847986041394, + "tokens_seen": 563347456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041882647943831497, + "loss": 2.9452, + "theoretical_loss": 3.8653373594846103, + "tokens_seen": 563412992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041881644934804415, + "loss": 2.8245, + "theoretical_loss": 3.8652899274277255, + "tokens_seen": 563478528 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041880641925777333, + "loss": 3.1132, + "theoretical_loss": 3.865242502431612, + "tokens_seen": 563544064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 321068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.633208751678467, + "objective/train/theoretical_loss": 3.865195084494398, + "objective/train/tokens_used": 584069600, + "theoretical_loss": 3.865195084494398, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187963891675025, + "loss": 2.6685, + "theoretical_loss": 3.865195084494398, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041878635907723175, + "loss": 2.5112, + "theoretical_loss": 3.8651476736142127, + "tokens_seen": 563675136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187763289869609, + "loss": 2.7993, + "theoretical_loss": 3.8651002697891848, + "tokens_seen": 563740672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187662988966901, + "loss": 2.7206, + "theoretical_loss": 3.8650528730174454, + "tokens_seen": 563806208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041875626880641924, + "loss": 2.8497, + "theoretical_loss": 3.8650054832971237, + "tokens_seen": 563871744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041874623871614847, + "loss": 2.6861, + "theoretical_loss": 3.8649581006263527, + "tokens_seen": 563937280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041873620862587765, + "loss": 2.8536, + "theoretical_loss": 3.8649107250032646, + "tokens_seen": 564002816 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041872617853560683, + "loss": 2.6021, + "theoretical_loss": 3.8648633564259915, + "tokens_seen": 564068352 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418716148445336, + "loss": 2.8927, + "theoretical_loss": 3.864815994892668, + "tokens_seen": 564133888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041870611835506525, + "loss": 2.6526, + "theoretical_loss": 3.8647686404014276, + "tokens_seen": 564199424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186960882647944, + "loss": 2.9926, + "theoretical_loss": 3.8647212929504065, + "tokens_seen": 564264960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186860581745236, + "loss": 2.8074, + "theoretical_loss": 3.8646739525377396, + "tokens_seen": 564330496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041867602808425274, + "loss": 2.8183, + "theoretical_loss": 3.864626619161564, + "tokens_seen": 564396032 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418665997993982, + "loss": 3.0095, + "theoretical_loss": 3.8645792928200167, + "tokens_seen": 564461568 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041865596790371115, + "loss": 2.9241, + "theoretical_loss": 3.8645319735112356, + "tokens_seen": 564527104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041864593781344034, + "loss": 2.8379, + "theoretical_loss": 3.8644846612333597, + "tokens_seen": 564592640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186359077231695, + "loss": 2.9107, + "theoretical_loss": 3.864437355984528, + "tokens_seen": 564658176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186258776328987, + "loss": 2.6939, + "theoretical_loss": 3.864390057762881, + "tokens_seen": 564723712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186158475426279, + "loss": 2.601, + "theoretical_loss": 3.8643427665665593, + "tokens_seen": 564789248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186058174523571, + "loss": 2.9561, + "theoretical_loss": 3.864295482393704, + "tokens_seen": 564854784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041859578736208624, + "loss": 2.6425, + "theoretical_loss": 3.8642482052424576, + "tokens_seen": 564920320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185857572718155, + "loss": 2.8036, + "theoretical_loss": 3.864200935110963, + "tokens_seen": 564985856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185757271815446, + "loss": 2.6034, + "theoretical_loss": 3.8641536719973644, + "tokens_seen": 565051392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041856569709127384, + "loss": 2.6046, + "theoretical_loss": 3.8641064158998057, + "tokens_seen": 565116928 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418555667001003, + "loss": 2.9108, + "theoretical_loss": 3.864059166816431, + "tokens_seen": 565182464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 321728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3612844944000244, + "objective/train/theoretical_loss": 3.8640119247453875, + "objective/train/tokens_used": 585708000, + "theoretical_loss": 3.8640119247453875, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185456369107322, + "loss": 2.8488, + "theoretical_loss": 3.8640119247453875, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185356068204614, + "loss": 2.7459, + "theoretical_loss": 3.863964689684821, + "tokens_seen": 565313536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185255767301906, + "loss": 2.7467, + "theoretical_loss": 3.8639174616328784, + "tokens_seen": 565379072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041851554663991974, + "loss": 2.5563, + "theoretical_loss": 3.863870240587708, + "tokens_seen": 565444608 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418505516549649, + "loss": 2.7328, + "theoretical_loss": 3.8638230265474585, + "tokens_seen": 565510144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184954864593781, + "loss": 2.6208, + "theoretical_loss": 3.863775819510278, + "tokens_seen": 565575680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041848545636910734, + "loss": 2.6409, + "theoretical_loss": 3.863728619474318, + "tokens_seen": 565641216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184754262788365, + "loss": 2.8141, + "theoretical_loss": 3.8636814264377284, + "tokens_seen": 565706752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184653961885657, + "loss": 2.6808, + "theoretical_loss": 3.8636342403986603, + "tokens_seen": 565772288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184553660982949, + "loss": 2.676, + "theoretical_loss": 3.8635870613552665, + "tokens_seen": 565837824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041844533600802407, + "loss": 2.9051, + "theoretical_loss": 3.863539889305699, + "tokens_seen": 565903360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041843530591775325, + "loss": 2.8133, + "theoretical_loss": 3.863492724248112, + "tokens_seen": 565968896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184252758274825, + "loss": 2.7888, + "theoretical_loss": 3.8634455661806593, + "tokens_seen": 566034432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184152457372116, + "loss": 2.9215, + "theoretical_loss": 3.8633984151014955, + "tokens_seen": 566099968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041840521564694085, + "loss": 2.8133, + "theoretical_loss": 3.863351271008777, + "tokens_seen": 566165504 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041839518555666997, + "loss": 2.8181, + "theoretical_loss": 3.863304133900659, + "tokens_seen": 566231040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183851554663992, + "loss": 2.6397, + "theoretical_loss": 3.863257003775299, + "tokens_seen": 566296576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183751253761284, + "loss": 3.028, + "theoretical_loss": 3.8632098806308557, + "tokens_seen": 566362112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041836509528585757, + "loss": 2.5726, + "theoretical_loss": 3.8631627644654856, + "tokens_seen": 566427648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041835506519558675, + "loss": 2.7655, + "theoretical_loss": 3.8631156552773493, + "tokens_seen": 566493184 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418345035105316, + "loss": 2.7821, + "theoretical_loss": 3.8630685530646054, + "tokens_seen": 566558720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183350050150451, + "loss": 2.9222, + "theoretical_loss": 3.8630214578254156, + "tokens_seen": 566624256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041832497492477435, + "loss": 2.7699, + "theoretical_loss": 3.8629743695579393, + "tokens_seen": 566689792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183149448345035, + "loss": 2.8905, + "theoretical_loss": 3.86292728826034, + "tokens_seen": 566755328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183049147442327, + "loss": 2.8477, + "theoretical_loss": 3.8628802139307803, + "tokens_seen": 566820864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 322867, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6280651092529297, + "objective/train/theoretical_loss": 3.8628331465674224, + "objective/train/tokens_used": 587346400, + "theoretical_loss": 3.8628331465674224, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182948846539619, + "loss": 2.8288, + "theoretical_loss": 3.8628331465674224, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182848545636911, + "loss": 2.683, + "theoretical_loss": 3.8627860861684304, + "tokens_seen": 566951936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041827482447342025, + "loss": 2.6811, + "theoretical_loss": 3.8627390327319695, + "tokens_seen": 567017472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041826479438314944, + "loss": 2.9301, + "theoretical_loss": 3.8626919862562055, + "tokens_seen": 567083008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182547642928786, + "loss": 2.6402, + "theoretical_loss": 3.8626449467393034, + "tokens_seen": 567148544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041824473420260785, + "loss": 2.9405, + "theoretical_loss": 3.86259791417943, + "tokens_seen": 567214080 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418234704112337, + "loss": 2.5639, + "theoretical_loss": 3.862550888574753, + "tokens_seen": 567279616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182246740220662, + "loss": 2.9047, + "theoretical_loss": 3.862503869923441, + "tokens_seen": 567345152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182146439317954, + "loss": 2.5505, + "theoretical_loss": 3.8624568582236622, + "tokens_seen": 567410688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182046138415246, + "loss": 2.625, + "theoretical_loss": 3.8624098534735865, + "tokens_seen": 567476224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041819458375125376, + "loss": 2.9179, + "theoretical_loss": 3.8623628556713836, + "tokens_seen": 567541760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041818455366098294, + "loss": 2.8786, + "theoretical_loss": 3.8623158648152254, + "tokens_seen": 567607296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181745235707121, + "loss": 2.6793, + "theoretical_loss": 3.862268880903282, + "tokens_seen": 567672832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041816449348044136, + "loss": 2.6765, + "theoretical_loss": 3.8622219039337273, + "tokens_seen": 567738368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041815446339017054, + "loss": 3.0911, + "theoretical_loss": 3.8621749339047335, + "tokens_seen": 567803904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181444332998997, + "loss": 2.6648, + "theoretical_loss": 3.862127970814474, + "tokens_seen": 567869440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181344032096289, + "loss": 2.6876, + "theoretical_loss": 3.8620810146611233, + "tokens_seen": 567934976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181243731193581, + "loss": 3.0197, + "theoretical_loss": 3.862034065442857, + "tokens_seen": 568000512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181143430290873, + "loss": 2.8192, + "theoretical_loss": 3.8619871231578498, + "tokens_seen": 568066048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041810431293881644, + "loss": 2.8597, + "theoretical_loss": 3.8619401878042794, + "tokens_seen": 568131584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180942828485457, + "loss": 2.7689, + "theoretical_loss": 3.861893259380322, + "tokens_seen": 568197120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180842527582748, + "loss": 2.9388, + "theoretical_loss": 3.8618463378841557, + "tokens_seen": 568262656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041807422266800404, + "loss": 2.7452, + "theoretical_loss": 3.861799423313959, + "tokens_seen": 568328192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180641925777332, + "loss": 2.7699, + "theoretical_loss": 3.861752515667911, + "tokens_seen": 568393728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180541624874624, + "loss": 2.7727, + "theoretical_loss": 3.861705614944192, + "tokens_seen": 568459264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 324005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.827101230621338, + "objective/train/theoretical_loss": 3.8616587211409823, + "objective/train/tokens_used": 588984800, + "theoretical_loss": 3.8616587211409823, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180441323971916, + "loss": 2.78, + "theoretical_loss": 3.8616587211409823, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180341023069208, + "loss": 2.9464, + "theoretical_loss": 3.861611834256463, + "tokens_seen": 568590336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041802407221664994, + "loss": 2.7669, + "theoretical_loss": 3.8615649542888164, + "tokens_seen": 568655872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180140421263792, + "loss": 2.9159, + "theoretical_loss": 3.861518081236224, + "tokens_seen": 568721408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180040120361083, + "loss": 2.6733, + "theoretical_loss": 3.861471215096871, + "tokens_seen": 568786944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041799398194583754, + "loss": 2.6195, + "theoretical_loss": 3.86142435586894, + "tokens_seen": 568852480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179839518555667, + "loss": 2.5565, + "theoretical_loss": 3.8613775035506164, + "tokens_seen": 568918016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179739217652959, + "loss": 2.9737, + "theoretical_loss": 3.8613306581400852, + "tokens_seen": 568983552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179638916750251, + "loss": 2.9827, + "theoretical_loss": 3.861283819635532, + "tokens_seen": 569049088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041795386158475427, + "loss": 2.5987, + "theoretical_loss": 3.861236988035145, + "tokens_seen": 569114624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041794383149448345, + "loss": 2.8994, + "theoretical_loss": 3.86119016333711, + "tokens_seen": 569180160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179338014042127, + "loss": 2.9604, + "theoretical_loss": 3.861143345539616, + "tokens_seen": 569245696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179237713139418, + "loss": 2.5996, + "theoretical_loss": 3.861096534640852, + "tokens_seen": 569311232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041791374122367105, + "loss": 2.7902, + "theoretical_loss": 3.8610497306390066, + "tokens_seen": 569376768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041790371113340017, + "loss": 2.6739, + "theoretical_loss": 3.861002933532271, + "tokens_seen": 569442304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178936810431294, + "loss": 2.668, + "theoretical_loss": 3.860956143318835, + "tokens_seen": 569507840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178836509528586, + "loss": 2.9281, + "theoretical_loss": 3.8609093599968913, + "tokens_seen": 569573376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041787362086258777, + "loss": 2.9559, + "theoretical_loss": 3.8608625835646313, + "tokens_seen": 569638912 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041786359077231695, + "loss": 2.6292, + "theoretical_loss": 3.860815814020248, + "tokens_seen": 569704448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178535606820462, + "loss": 2.8555, + "theoretical_loss": 3.860769051361935, + "tokens_seen": 569769984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178435305917753, + "loss": 2.6426, + "theoretical_loss": 3.8607222955878866, + "tokens_seen": 569835520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783350050150455, + "loss": 2.7501, + "theoretical_loss": 3.860675546696298, + "tokens_seen": 569901056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178234704112337, + "loss": 2.9662, + "theoretical_loss": 3.8606288046853647, + "tokens_seen": 569966592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178134403209629, + "loss": 2.7383, + "theoretical_loss": 3.8605820695532826, + "tokens_seen": 570032128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178034102306921, + "loss": 2.6567, + "theoretical_loss": 3.860535341298249, + "tokens_seen": 570097664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 324627, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.945570468902588, + "objective/train/theoretical_loss": 3.860488619918462, + "objective/train/tokens_used": 590623200, + "theoretical_loss": 3.860488619918462, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177933801404213, + "loss": 2.8017, + "theoretical_loss": 3.860488619918462, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041778335005015045, + "loss": 3.0509, + "theoretical_loss": 3.8604419054121193, + "tokens_seen": 570228736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041777331995987964, + "loss": 2.8242, + "theoretical_loss": 3.86039519777742, + "tokens_seen": 570294272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177632898696088, + "loss": 2.59, + "theoretical_loss": 3.8603484970125637, + "tokens_seen": 570359808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041775325977933805, + "loss": 2.8078, + "theoretical_loss": 3.8603018031157514, + "tokens_seen": 570425344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177432296890672, + "loss": 3.0908, + "theoretical_loss": 3.8602551160851837, + "tokens_seen": 570490880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177331995987964, + "loss": 2.8318, + "theoretical_loss": 3.8602084359190627, + "tokens_seen": 570556416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177231695085256, + "loss": 2.9725, + "theoretical_loss": 3.86016176261559, + "tokens_seen": 570621952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177131394182548, + "loss": 2.67, + "theoretical_loss": 3.8601150961729696, + "tokens_seen": 570687488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041770310932798396, + "loss": 2.6319, + "theoretical_loss": 3.8600684365894047, + "tokens_seen": 570753024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041769307923771314, + "loss": 2.8908, + "theoretical_loss": 3.8600217838631, + "tokens_seen": 570818560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176830491474423, + "loss": 2.7601, + "theoretical_loss": 3.85997513799226, + "tokens_seen": 570884096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041767301905717156, + "loss": 2.9639, + "theoretical_loss": 3.859928498975092, + "tokens_seen": 570949632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176629889669007, + "loss": 2.9724, + "theoretical_loss": 3.859881866809801, + "tokens_seen": 571015168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176529588766299, + "loss": 2.8134, + "theoretical_loss": 3.859835241494595, + "tokens_seen": 571080704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041764292878635904, + "loss": 2.7331, + "theoretical_loss": 3.859788623027681, + "tokens_seen": 571146240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176328986960883, + "loss": 2.6944, + "theoretical_loss": 3.859742011407268, + "tokens_seen": 571211776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041762286860581746, + "loss": 2.8846, + "theoretical_loss": 3.859695406631565, + "tokens_seen": 571277312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041761283851554664, + "loss": 2.7068, + "theoretical_loss": 3.8596488086987826, + "tokens_seen": 571342848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176028084252758, + "loss": 2.7232, + "theoretical_loss": 3.8596022176071303, + "tokens_seen": 571408384 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417592778335005, + "loss": 2.7892, + "theoretical_loss": 3.8595556333548195, + "tokens_seen": 571473920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175827482447342, + "loss": 2.4485, + "theoretical_loss": 3.859509055940063, + "tokens_seen": 571539456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175727181544634, + "loss": 2.5149, + "theoretical_loss": 3.859462485361072, + "tokens_seen": 571604992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041756268806419255, + "loss": 2.6744, + "theoretical_loss": 3.85941592161606, + "tokens_seen": 571670528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175526579739218, + "loss": 2.7721, + "theoretical_loss": 3.859369364703242, + "tokens_seen": 571736064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 325757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.774052619934082, + "objective/train/theoretical_loss": 3.8593228146208314, + "objective/train/tokens_used": 592261600, + "theoretical_loss": 3.8593228146208314, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041754262788365096, + "loss": 2.8318, + "theoretical_loss": 3.8593228146208314, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041753259779338015, + "loss": 2.966, + "theoretical_loss": 3.859276271367044, + "tokens_seen": 571867136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175225677031093, + "loss": 2.7908, + "theoretical_loss": 3.859229734940095, + "tokens_seen": 571932672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175125376128385, + "loss": 2.5661, + "theoretical_loss": 3.8591832053382014, + "tokens_seen": 571998208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175025075225677, + "loss": 2.7081, + "theoretical_loss": 3.8591366825595808, + "tokens_seen": 572063744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174924774322969, + "loss": 2.621, + "theoretical_loss": 3.85909016660245, + "tokens_seen": 572129280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041748244734202605, + "loss": 2.8749, + "theoretical_loss": 3.8590436574650293, + "tokens_seen": 572194816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174724172517553, + "loss": 2.705, + "theoretical_loss": 3.858997155145537, + "tokens_seen": 572260352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174623871614844, + "loss": 2.7483, + "theoretical_loss": 3.858950659642192, + "tokens_seen": 572325888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041745235707121365, + "loss": 2.7732, + "theoretical_loss": 3.8589041709532164, + "tokens_seen": 572391424 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041744232698094283, + "loss": 2.5971, + "theoretical_loss": 3.8588576890768307, + "tokens_seen": 572456960 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417432296890672, + "loss": 2.672, + "theoretical_loss": 3.858811214011258, + "tokens_seen": 572522496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174222668004012, + "loss": 2.8971, + "theoretical_loss": 3.8587647457547183, + "tokens_seen": 572588032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041741223671013037, + "loss": 2.6925, + "theoretical_loss": 3.8587182843054375, + "tokens_seen": 572653568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174022066198596, + "loss": 2.6789, + "theoretical_loss": 3.8586718296616382, + "tokens_seen": 572719104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173921765295888, + "loss": 2.8008, + "theoretical_loss": 3.858625381821545, + "tokens_seen": 572784640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041738214643931797, + "loss": 2.7738, + "theoretical_loss": 3.858578940783384, + "tokens_seen": 572850176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041737211634904715, + "loss": 2.721, + "theoretical_loss": 3.85853250654538, + "tokens_seen": 572915712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173620862587764, + "loss": 2.7751, + "theoretical_loss": 3.8584860791057602, + "tokens_seen": 572981248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173520561685055, + "loss": 2.5138, + "theoretical_loss": 3.8584396584627516, + "tokens_seen": 573046784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041734202607823475, + "loss": 2.7566, + "theoretical_loss": 3.8583932446145828, + "tokens_seen": 573112320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173319959879639, + "loss": 2.808, + "theoretical_loss": 3.8583468375594814, + "tokens_seen": 573177856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173219658976931, + "loss": 2.7426, + "theoretical_loss": 3.858300437295677, + "tokens_seen": 573243392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173119358074223, + "loss": 2.7867, + "theoretical_loss": 3.8582540438214, + "tokens_seen": 573308928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173019057171515, + "loss": 2.8751, + "theoretical_loss": 3.85820765713488, + "tokens_seen": 573374464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 326467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8871238231658936, + "objective/train/theoretical_loss": 3.858161277234349, + "objective/train/tokens_used": 593900000, + "theoretical_loss": 3.858161277234349, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041729187562688065, + "loss": 2.8817, + "theoretical_loss": 3.858161277234349, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041728184553660984, + "loss": 2.7829, + "theoretical_loss": 3.8581149041180387, + "tokens_seen": 573505536 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417271815446339, + "loss": 2.7302, + "theoretical_loss": 3.858068537784182, + "tokens_seen": 573571072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041726178535606825, + "loss": 2.8625, + "theoretical_loss": 3.8580221782310113, + "tokens_seen": 573636608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172517552657974, + "loss": 2.8321, + "theoretical_loss": 3.8579758254567613, + "tokens_seen": 573702144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172417251755266, + "loss": 2.7675, + "theoretical_loss": 3.8579294794596657, + "tokens_seen": 573767680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172316950852558, + "loss": 2.8152, + "theoretical_loss": 3.85788314023796, + "tokens_seen": 573833216 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417221664994985, + "loss": 2.8103, + "theoretical_loss": 3.857836807789881, + "tokens_seen": 573898752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041721163490471416, + "loss": 2.833, + "theoretical_loss": 3.857790482113664, + "tokens_seen": 573964288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041720160481444334, + "loss": 2.886, + "theoretical_loss": 3.8577441632075473, + "tokens_seen": 574029824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171915747241725, + "loss": 2.9433, + "theoretical_loss": 3.8576978510697675, + "tokens_seen": 574095360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041718154463390176, + "loss": 2.8847, + "theoretical_loss": 3.8576515456985634, + "tokens_seen": 574160896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171715145436309, + "loss": 2.6894, + "theoretical_loss": 3.857605247092175, + "tokens_seen": 574226432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171614844533601, + "loss": 2.6527, + "theoretical_loss": 3.857558955248842, + "tokens_seen": 574291968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041715145436308924, + "loss": 2.6139, + "theoretical_loss": 3.8575126701668037, + "tokens_seen": 574357504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171414242728185, + "loss": 2.9821, + "theoretical_loss": 3.8574663918443024, + "tokens_seen": 574423040 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041713139418254766, + "loss": 2.7302, + "theoretical_loss": 3.857420120279579, + "tokens_seen": 574488576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041712136409227684, + "loss": 2.6908, + "theoretical_loss": 3.857373855470877, + "tokens_seen": 574554112 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417111334002006, + "loss": 2.8465, + "theoretical_loss": 3.8573275974164387, + "tokens_seen": 574619648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171013039117352, + "loss": 2.6284, + "theoretical_loss": 3.8572813461145086, + "tokens_seen": 574685184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170912738214644, + "loss": 2.7787, + "theoretical_loss": 3.8572351015633304, + "tokens_seen": 574750720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170812437311936, + "loss": 2.6762, + "theoretical_loss": 3.8571888637611496, + "tokens_seen": 574816256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041707121364092275, + "loss": 2.9144, + "theoretical_loss": 3.857142632706212, + "tokens_seen": 574881792 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417061183550652, + "loss": 2.8601, + "theoretical_loss": 3.857096408396764, + "tokens_seen": 574947328 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041705115346038116, + "loss": 2.6951, + "theoretical_loss": 3.8570501908310515, + "tokens_seen": 575012864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 327672, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.893551826477051, + "objective/train/theoretical_loss": 3.857003980007324, + "objective/train/tokens_used": 595538400, + "theoretical_loss": 3.857003980007324, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041704112337011035, + "loss": 2.8592, + "theoretical_loss": 3.857003980007324, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170310932798395, + "loss": 2.7133, + "theoretical_loss": 3.8569577759238287, + "tokens_seen": 575143936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170210631895687, + "loss": 2.7563, + "theoretical_loss": 3.8569115785788153, + "tokens_seen": 575209472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170110330992979, + "loss": 2.9113, + "theoretical_loss": 3.856865387970533, + "tokens_seen": 575275008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170010030090271, + "loss": 2.8067, + "theoretical_loss": 3.8568192040972322, + "tokens_seen": 575340544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041699097291875625, + "loss": 2.5334, + "theoretical_loss": 3.856773026957164, + "tokens_seen": 575406080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169809428284855, + "loss": 2.765, + "theoretical_loss": 3.8567268565485806, + "tokens_seen": 575471616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169709127382146, + "loss": 2.7038, + "theoretical_loss": 3.856680692869733, + "tokens_seen": 575537152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041696088264794385, + "loss": 2.8842, + "theoretical_loss": 3.856634535918875, + "tokens_seen": 575602688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041695085255767303, + "loss": 2.7644, + "theoretical_loss": 3.85658838569426, + "tokens_seen": 575668224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169408224674022, + "loss": 2.7948, + "theoretical_loss": 3.856542242194142, + "tokens_seen": 575733760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169307923771314, + "loss": 2.7855, + "theoretical_loss": 3.8564961054167766, + "tokens_seen": 575799296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169207622868606, + "loss": 2.8138, + "theoretical_loss": 3.8564499753604187, + "tokens_seen": 575864832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041691073219658975, + "loss": 2.8032, + "theoretical_loss": 3.856403852023325, + "tokens_seen": 575930368 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416900702106319, + "loss": 2.9167, + "theoretical_loss": 3.8563577354037513, + "tokens_seen": 575995904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168906720160481, + "loss": 3.0506, + "theoretical_loss": 3.8563116254999565, + "tokens_seen": 576061440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041688064192577735, + "loss": 3.0837, + "theoretical_loss": 3.856265522310198, + "tokens_seen": 576126976 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041687061183550653, + "loss": 2.6939, + "theoretical_loss": 3.8562194258327347, + "tokens_seen": 576192512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168605817452357, + "loss": 2.6713, + "theoretical_loss": 3.856173336065826, + "tokens_seen": 576258048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168505516549649, + "loss": 2.8135, + "theoretical_loss": 3.856127253007732, + "tokens_seen": 576323584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168405215646941, + "loss": 2.6818, + "theoretical_loss": 3.8560811766567133, + "tokens_seen": 576389120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041683049147442326, + "loss": 2.9436, + "theoretical_loss": 3.856035107011032, + "tokens_seen": 576454656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168204613841525, + "loss": 2.7948, + "theoretical_loss": 3.8559890440689495, + "tokens_seen": 576520192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168104312938816, + "loss": 2.7917, + "theoretical_loss": 3.855942987828728, + "tokens_seen": 576585728 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041680040120361085, + "loss": 2.6469, + "theoretical_loss": 3.855896938288632, + "tokens_seen": 576651264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 328706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.591416597366333, + "objective/train/theoretical_loss": 3.855850895446925, + "objective/train/tokens_used": 597176800, + "theoretical_loss": 3.855850895446925, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041679037111334, + "loss": 2.5328, + "theoretical_loss": 3.855850895446925, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167803410230692, + "loss": 2.7952, + "theoretical_loss": 3.8558048593018714, + "tokens_seen": 576782336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167703109327984, + "loss": 2.6321, + "theoretical_loss": 3.855758829851737, + "tokens_seen": 576847872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167602808425276, + "loss": 2.7992, + "theoretical_loss": 3.855712807094787, + "tokens_seen": 576913408 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041675025075225676, + "loss": 2.8481, + "theoretical_loss": 3.8556667910292886, + "tokens_seen": 576978944 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416740220661986, + "loss": 2.6563, + "theoretical_loss": 3.855620781653509, + "tokens_seen": 577044480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167301905717151, + "loss": 2.8306, + "theoretical_loss": 3.8555747789657158, + "tokens_seen": 577110016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041672016048144436, + "loss": 2.8782, + "theoretical_loss": 3.855528782964177, + "tokens_seen": 577175552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167101303911735, + "loss": 2.6459, + "theoretical_loss": 3.8554827936471625, + "tokens_seen": 577241088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167001003009027, + "loss": 2.6478, + "theoretical_loss": 3.855436811012942, + "tokens_seen": 577306624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166900702106319, + "loss": 3.059, + "theoretical_loss": 3.8553908350597865, + "tokens_seen": 577372160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166800401203611, + "loss": 2.7494, + "theoretical_loss": 3.855344865785966, + "tokens_seen": 577437696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041667001003009026, + "loss": 2.936, + "theoretical_loss": 3.8552989031897527, + "tokens_seen": 577503232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041665997993981944, + "loss": 2.4975, + "theoretical_loss": 3.855252947269419, + "tokens_seen": 577568768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166499498495487, + "loss": 2.5769, + "theoretical_loss": 3.8552069980232377, + "tokens_seen": 577634304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041663991975927786, + "loss": 2.8171, + "theoretical_loss": 3.855161055449483, + "tokens_seen": 577699840 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041662988966900704, + "loss": 2.7957, + "theoretical_loss": 3.8551151195464284, + "tokens_seen": 577765376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166198595787362, + "loss": 2.8499, + "theoretical_loss": 3.8550691903123493, + "tokens_seen": 577830912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166098294884654, + "loss": 2.7152, + "theoretical_loss": 3.855023267745522, + "tokens_seen": 577896448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165997993981946, + "loss": 2.7041, + "theoretical_loss": 3.854977351844221, + "tokens_seen": 577961984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165897693079238, + "loss": 2.7119, + "theoretical_loss": 3.8549314426067243, + "tokens_seen": 578027520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041657973921765295, + "loss": 2.9118, + "theoretical_loss": 3.85488554003131, + "tokens_seen": 578093056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165697091273822, + "loss": 2.7783, + "theoretical_loss": 3.854839644116255, + "tokens_seen": 578158592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041655967903711136, + "loss": 2.8074, + "theoretical_loss": 3.8547937548598386, + "tokens_seen": 578224128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654964894684055, + "loss": 2.9678, + "theoretical_loss": 3.8547478722603405, + "tokens_seen": 578289664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 329258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3565433025360107, + "objective/train/theoretical_loss": 3.8547019963160407, + "objective/train/tokens_used": 598815200, + "theoretical_loss": 3.8547019963160407, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165396188565697, + "loss": 2.7488, + "theoretical_loss": 3.8547019963160407, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165295887662989, + "loss": 2.957, + "theoretical_loss": 3.854656127025219, + "tokens_seen": 578420736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165195586760281, + "loss": 2.5597, + "theoretical_loss": 3.854610264386158, + "tokens_seen": 578486272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165095285857573, + "loss": 2.8326, + "theoretical_loss": 3.854564408397139, + "tokens_seen": 578551808 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041649949849548645, + "loss": 2.8244, + "theoretical_loss": 3.854518559056445, + "tokens_seen": 578617344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164894684052157, + "loss": 2.7684, + "theoretical_loss": 3.8544727163623587, + "tokens_seen": 578682880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164794383149448, + "loss": 2.878, + "theoretical_loss": 3.8544268803131643, + "tokens_seen": 578748416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041646940822467405, + "loss": 2.7852, + "theoretical_loss": 3.8543810509071466, + "tokens_seen": 578813952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041645937813440323, + "loss": 2.7624, + "theoretical_loss": 3.8543352281425904, + "tokens_seen": 578879488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164493480441324, + "loss": 2.6575, + "theoretical_loss": 3.8542894120177817, + "tokens_seen": 578945024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164393179538616, + "loss": 2.8381, + "theoretical_loss": 3.8542436025310067, + "tokens_seen": 579010560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164292878635908, + "loss": 2.6767, + "theoretical_loss": 3.854197799680553, + "tokens_seen": 579076096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041641925777331995, + "loss": 2.9494, + "theoretical_loss": 3.854152003464708, + "tokens_seen": 579141632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164092276830492, + "loss": 2.8045, + "theoretical_loss": 3.85410621388176, + "tokens_seen": 579207168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163991975927783, + "loss": 2.8555, + "theoretical_loss": 3.854060430929998, + "tokens_seen": 579272704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638916750250755, + "loss": 3.0175, + "theoretical_loss": 3.854014654607712, + "tokens_seen": 579338240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041637913741223673, + "loss": 2.8226, + "theoretical_loss": 3.8539688849131917, + "tokens_seen": 579403776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163691073219659, + "loss": 3.0263, + "theoretical_loss": 3.853923121844728, + "tokens_seen": 579469312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163590772316951, + "loss": 2.6421, + "theoretical_loss": 3.8538773654006127, + "tokens_seen": 579534848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163490471414243, + "loss": 2.8617, + "theoretical_loss": 3.8538316155791383, + "tokens_seen": 579600384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041633901705115346, + "loss": 2.6148, + "theoretical_loss": 3.853785872378597, + "tokens_seen": 579665920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163289869608827, + "loss": 2.7086, + "theoretical_loss": 3.853740135797282, + "tokens_seen": 579731456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163189568706118, + "loss": 2.9555, + "theoretical_loss": 3.8536944058334885, + "tokens_seen": 579796992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041630892678034105, + "loss": 2.782, + "theoretical_loss": 3.85364868248551, + "tokens_seen": 579862528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162988966900702, + "loss": 3.079, + "theoretical_loss": 3.8536029657516426, + "tokens_seen": 579928064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 329612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.266913890838623, + "objective/train/theoretical_loss": 3.8535572556301823, + "objective/train/tokens_used": 600453600, + "theoretical_loss": 3.8535572556301823, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162888665997994, + "loss": 2.9089, + "theoretical_loss": 3.8535572556301823, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162788365095286, + "loss": 2.8912, + "theoretical_loss": 3.8535115521194245, + "tokens_seen": 580059136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162688064192578, + "loss": 2.857, + "theoretical_loss": 3.8534658552176677, + "tokens_seen": 580124672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041625877632898696, + "loss": 2.8663, + "theoretical_loss": 3.853420164923209, + "tokens_seen": 580190208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162487462387162, + "loss": 2.6089, + "theoretical_loss": 3.853374481234348, + "tokens_seen": 580255744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162387161484453, + "loss": 3.0777, + "theoretical_loss": 3.8533288041493825, + "tokens_seen": 580321280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041622868605817456, + "loss": 2.7666, + "theoretical_loss": 3.853283133666613, + "tokens_seen": 580386816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162186559679037, + "loss": 2.8469, + "theoretical_loss": 3.8532374697843395, + "tokens_seen": 580452352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162086258776329, + "loss": 3.1069, + "theoretical_loss": 3.853191812500863, + "tokens_seen": 580517888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161985957873621, + "loss": 2.5315, + "theoretical_loss": 3.853146161814485, + "tokens_seen": 580583424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161885656970913, + "loss": 3.1341, + "theoretical_loss": 3.853100517723509, + "tokens_seen": 580648960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041617853560682046, + "loss": 2.8452, + "theoretical_loss": 3.853054880226236, + "tokens_seen": 580714496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041616850551654964, + "loss": 2.642, + "theoretical_loss": 3.8530092493209707, + "tokens_seen": 580780032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161584754262788, + "loss": 2.9064, + "theoretical_loss": 3.8529636250060166, + "tokens_seen": 580845568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041614844533600806, + "loss": 2.8612, + "theoretical_loss": 3.85291800727968, + "tokens_seen": 580911104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161384152457372, + "loss": 3.071, + "theoretical_loss": 3.8528723961402638, + "tokens_seen": 580976640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161283851554664, + "loss": 2.9736, + "theoretical_loss": 3.852826791586076, + "tokens_seen": 581042176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041611835506519555, + "loss": 2.659, + "theoretical_loss": 3.8527811936154217, + "tokens_seen": 581107712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161083249749248, + "loss": 2.744, + "theoretical_loss": 3.8527356022266104, + "tokens_seen": 581173248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041609829488465397, + "loss": 2.8711, + "theoretical_loss": 3.852690017417948, + "tokens_seen": 581238784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041608826479438315, + "loss": 3.0193, + "theoretical_loss": 3.8526444391877437, + "tokens_seen": 581304320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041607823470411233, + "loss": 2.7875, + "theoretical_loss": 3.852598867534307, + "tokens_seen": 581369856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041606820461384156, + "loss": 2.6953, + "theoretical_loss": 3.852553302455947, + "tokens_seen": 581435392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160581745235707, + "loss": 3.0129, + "theoretical_loss": 3.8525077439509747, + "tokens_seen": 581500928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160481444332999, + "loss": 2.7702, + "theoretical_loss": 3.8524621920177005, + "tokens_seen": 581566464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 330311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.916626453399658, + "objective/train/theoretical_loss": 3.852416646654437, + "objective/train/tokens_used": 602092000, + "theoretical_loss": 3.852416646654437, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041603811434302905, + "loss": 2.8318, + "theoretical_loss": 3.852416646654437, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160280842527583, + "loss": 2.5899, + "theoretical_loss": 3.8523711078594953, + "tokens_seen": 581697536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041601805416248747, + "loss": 2.9073, + "theoretical_loss": 3.8523255756311894, + "tokens_seen": 581763072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041600802407221665, + "loss": 2.9442, + "theoretical_loss": 3.852280049967832, + "tokens_seen": 581828608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041599799398194583, + "loss": 2.9509, + "theoretical_loss": 3.8522345308677375, + "tokens_seen": 581894144 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415987963891675, + "loss": 2.5574, + "theoretical_loss": 3.8521890183292213, + "tokens_seen": 581959680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159779338014042, + "loss": 2.918, + "theoretical_loss": 3.8521435123505974, + "tokens_seen": 582025216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041596790371113343, + "loss": 2.8101, + "theoretical_loss": 3.8520980129301834, + "tokens_seen": 582090752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041595787362086256, + "loss": 2.611, + "theoretical_loss": 3.852052520066295, + "tokens_seen": 582156288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159478435305918, + "loss": 2.7954, + "theoretical_loss": 3.85200703375725, + "tokens_seen": 582221824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159378134403209, + "loss": 2.8191, + "theoretical_loss": 3.851961554001365, + "tokens_seen": 582287360 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041592778335005015, + "loss": 2.8662, + "theoretical_loss": 3.85191608079696, + "tokens_seen": 582352896 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041591775325977934, + "loss": 2.8765, + "theoretical_loss": 3.8518706141423538, + "tokens_seen": 582418432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159077231695085, + "loss": 2.9141, + "theoretical_loss": 3.851825154035865, + "tokens_seen": 582483968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041589769307923775, + "loss": 2.9011, + "theoretical_loss": 3.8517797004758156, + "tokens_seen": 582549504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041588766298896693, + "loss": 2.7864, + "theoretical_loss": 3.8517342534605254, + "tokens_seen": 582615040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158776328986961, + "loss": 2.6935, + "theoretical_loss": 3.851688812988317, + "tokens_seen": 582680576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158676028084253, + "loss": 2.9189, + "theoretical_loss": 3.8516433790575118, + "tokens_seen": 582746112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158575727181545, + "loss": 3.0231, + "theoretical_loss": 3.8515979516664327, + "tokens_seen": 582811648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041584754262788366, + "loss": 2.7779, + "theoretical_loss": 3.851552530813403, + "tokens_seen": 582877184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158375125376129, + "loss": 2.7526, + "theoretical_loss": 3.8515071164967476, + "tokens_seen": 582942720 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415827482447342, + "loss": 2.8509, + "theoretical_loss": 3.8514617087147904, + "tokens_seen": 583008256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041581745235707126, + "loss": 2.9512, + "theoretical_loss": 3.8514163074658576, + "tokens_seen": 583073792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158074222668004, + "loss": 2.8851, + "theoretical_loss": 3.851370912748274, + "tokens_seen": 583139328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157973921765296, + "loss": 2.9043, + "theoretical_loss": 3.851325524560367, + "tokens_seen": 583204864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 331192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3419594764709473, + "objective/train/theoretical_loss": 3.851280142900463, + "objective/train/tokens_used": 603730400, + "theoretical_loss": 3.851280142900463, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157873620862588, + "loss": 2.7418, + "theoretical_loss": 3.851280142900463, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415777331995988, + "loss": 2.8718, + "theoretical_loss": 3.8512347677668908, + "tokens_seen": 583335936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041576730190571716, + "loss": 2.7871, + "theoretical_loss": 3.851189399157978, + "tokens_seen": 583401472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157572718154464, + "loss": 2.6968, + "theoretical_loss": 3.8511440370720535, + "tokens_seen": 583467008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157472417251755, + "loss": 2.8935, + "theoretical_loss": 3.851098681507448, + "tokens_seen": 583532544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041573721163490476, + "loss": 2.6811, + "theoretical_loss": 3.8510533324624907, + "tokens_seen": 583598080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157271815446339, + "loss": 2.7665, + "theoretical_loss": 3.8510079899355123, + "tokens_seen": 583663616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157171514543631, + "loss": 3.0478, + "theoretical_loss": 3.8509626539248454, + "tokens_seen": 583729152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157071213640923, + "loss": 2.8275, + "theoretical_loss": 3.8509173244288206, + "tokens_seen": 583794688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156970912738215, + "loss": 2.7585, + "theoretical_loss": 3.8508720014457722, + "tokens_seen": 583860224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041568706118355066, + "loss": 2.8001, + "theoretical_loss": 3.8508266849740327, + "tokens_seen": 583925760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041567703109327984, + "loss": 2.9887, + "theoretical_loss": 3.8507813750119353, + "tokens_seen": 583991296 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415667001003009, + "loss": 2.8654, + "theoretical_loss": 3.8507360715578156, + "tokens_seen": 584056832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041565697091273826, + "loss": 2.9364, + "theoretical_loss": 3.8506907746100087, + "tokens_seen": 584122368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156469408224674, + "loss": 2.8208, + "theoretical_loss": 3.85064548416685, + "tokens_seen": 584187904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156369107321966, + "loss": 3.1207, + "theoretical_loss": 3.850600200226676, + "tokens_seen": 584253440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041562688064192575, + "loss": 2.9591, + "theoretical_loss": 3.8505549227878237, + "tokens_seen": 584318976 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415616850551655, + "loss": 2.8618, + "theoretical_loss": 3.8505096518486304, + "tokens_seen": 584384512 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041560682046138417, + "loss": 2.8048, + "theoretical_loss": 3.8504643874074347, + "tokens_seen": 584450048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041559679037111335, + "loss": 2.7938, + "theoretical_loss": 3.8504191294625754, + "tokens_seen": 584515584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041558676028084253, + "loss": 3.0779, + "theoretical_loss": 3.850373878012392, + "tokens_seen": 584581120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041557673019057176, + "loss": 2.727, + "theoretical_loss": 3.850328633055224, + "tokens_seen": 584646656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155667001003009, + "loss": 3.0438, + "theoretical_loss": 3.8502833945894124, + "tokens_seen": 584712192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155566700100301, + "loss": 2.9217, + "theoretical_loss": 3.850238162613299, + "tokens_seen": 584777728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041554663991975925, + "loss": 2.9176, + "theoretical_loss": 3.850192937125225, + "tokens_seen": 584843264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 331815, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.829080104827881, + "objective/train/theoretical_loss": 3.850147718123533, + "objective/train/tokens_used": 605368800, + "theoretical_loss": 3.850147718123533, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155366098294885, + "loss": 2.7848, + "theoretical_loss": 3.850147718123533, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041552657973921767, + "loss": 2.877, + "theoretical_loss": 3.850102505606566, + "tokens_seen": 584974336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041551654964894685, + "loss": 2.7364, + "theoretical_loss": 3.8500572995726676, + "tokens_seen": 585039872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041550651955867603, + "loss": 2.891, + "theoretical_loss": 3.850012100020183, + "tokens_seen": 585105408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154964894684052, + "loss": 2.779, + "theoretical_loss": 3.849966906947456, + "tokens_seen": 585170944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154864593781344, + "loss": 2.9868, + "theoretical_loss": 3.8499217203528335, + "tokens_seen": 585236480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041547642928786363, + "loss": 2.7509, + "theoretical_loss": 3.8498765402346597, + "tokens_seen": 585302016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041546639919759276, + "loss": 2.851, + "theoretical_loss": 3.8498313665912827, + "tokens_seen": 585367552 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415456369107322, + "loss": 2.8037, + "theoretical_loss": 3.84978619942105, + "tokens_seen": 585433088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154463390170511, + "loss": 2.8823, + "theoretical_loss": 3.8497410387223088, + "tokens_seen": 585498624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041543630892678035, + "loss": 3.0639, + "theoretical_loss": 3.849695884493408, + "tokens_seen": 585564160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041542627883650954, + "loss": 2.5332, + "theoretical_loss": 3.849650736732696, + "tokens_seen": 585629696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154162487462387, + "loss": 2.8753, + "theoretical_loss": 3.849605595438524, + "tokens_seen": 585695232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154062186559679, + "loss": 3.111, + "theoretical_loss": 3.8495604606092417, + "tokens_seen": 585760768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041539618856569713, + "loss": 2.9358, + "theoretical_loss": 3.8495153322432003, + "tokens_seen": 585826304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041538615847542626, + "loss": 2.7838, + "theoretical_loss": 3.849470210338751, + "tokens_seen": 585891840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153761283851555, + "loss": 2.8008, + "theoretical_loss": 3.849425094894246, + "tokens_seen": 585957376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153660982948846, + "loss": 2.8761, + "theoretical_loss": 3.8493799859080378, + "tokens_seen": 586022912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041535606820461386, + "loss": 2.9351, + "theoretical_loss": 3.8493348833784813, + "tokens_seen": 586088448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041534603811434304, + "loss": 2.8079, + "theoretical_loss": 3.849289787303929, + "tokens_seen": 586153984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153360080240722, + "loss": 2.8135, + "theoretical_loss": 3.8492446976827357, + "tokens_seen": 586219520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153259779338014, + "loss": 2.7149, + "theoretical_loss": 3.8491996145132568, + "tokens_seen": 586285056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153159478435306, + "loss": 2.7679, + "theoretical_loss": 3.8491545377938485, + "tokens_seen": 586350592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041530591775325976, + "loss": 2.5723, + "theoretical_loss": 3.849109467522867, + "tokens_seen": 586416128 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415295887662989, + "loss": 2.7364, + "theoretical_loss": 3.8490644036986694, + "tokens_seen": 586481664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 333131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1692469120025635, + "objective/train/theoretical_loss": 3.8490193463196127, + "objective/train/tokens_used": 607007200, + "theoretical_loss": 3.8490193463196127, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152858575727181, + "loss": 2.6856, + "theoretical_loss": 3.8490193463196127, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041527582748244736, + "loss": 2.8743, + "theoretical_loss": 3.848974295384056, + "tokens_seen": 586612736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152657973921765, + "loss": 2.8334, + "theoretical_loss": 3.8489292508903574, + "tokens_seen": 586678272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152557673019057, + "loss": 2.7919, + "theoretical_loss": 3.848884212836877, + "tokens_seen": 586743808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152457372116349, + "loss": 2.8642, + "theoretical_loss": 3.848839181221974, + "tokens_seen": 586809344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152357071213641, + "loss": 2.7809, + "theoretical_loss": 3.84879415604401, + "tokens_seen": 586874880 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041522567703109327, + "loss": 2.907, + "theoretical_loss": 3.8487491373013456, + "tokens_seen": 586940416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152156469408225, + "loss": 2.6461, + "theoretical_loss": 3.848704124992343, + "tokens_seen": 587005952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041520561685055163, + "loss": 2.59, + "theoretical_loss": 3.8486591191153643, + "tokens_seen": 587071488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041519558676028086, + "loss": 2.8257, + "theoretical_loss": 3.848614119668772, + "tokens_seen": 587137024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041518555667001, + "loss": 2.8721, + "theoretical_loss": 3.8485691266509314, + "tokens_seen": 587202560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151755265797392, + "loss": 3.1213, + "theoretical_loss": 3.8485241400602055, + "tokens_seen": 587268096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151654964894684, + "loss": 2.6248, + "theoretical_loss": 3.848479159894959, + "tokens_seen": 587333632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151554663991976, + "loss": 2.8182, + "theoretical_loss": 3.8484341861535576, + "tokens_seen": 587399168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151454363089268, + "loss": 2.7422, + "theoretical_loss": 3.8483892188343676, + "tokens_seen": 587464704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041513540621865595, + "loss": 2.9843, + "theoretical_loss": 3.8483442579357554, + "tokens_seen": 587530240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151253761283852, + "loss": 2.7539, + "theoretical_loss": 3.8482993034560877, + "tokens_seen": 587595776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041511534603811437, + "loss": 2.876, + "theoretical_loss": 3.848254355393734, + "tokens_seen": 587661312 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041510531594784355, + "loss": 2.7914, + "theoretical_loss": 3.8482094137470604, + "tokens_seen": 587726848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041509528585757273, + "loss": 2.9117, + "theoretical_loss": 3.8481644785144375, + "tokens_seen": 587792384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041508525576730196, + "loss": 2.741, + "theoretical_loss": 3.8481195496942346, + "tokens_seen": 587857920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150752256770311, + "loss": 2.6959, + "theoretical_loss": 3.848074627284822, + "tokens_seen": 587923456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150651955867603, + "loss": 2.7976, + "theoretical_loss": 3.84802971128457, + "tokens_seen": 587988992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041505516549648945, + "loss": 2.8536, + "theoretical_loss": 3.84798480169185, + "tokens_seen": 588054528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150451354062187, + "loss": 2.9974, + "theoretical_loss": 3.8479398985050346, + "tokens_seen": 588120064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 334325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.72865891456604, + "objective/train/theoretical_loss": 3.847895001722496, + "objective/train/tokens_used": 608645600, + "theoretical_loss": 3.847895001722496, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041503510531594787, + "loss": 2.8432, + "theoretical_loss": 3.847895001722496, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041502507522567705, + "loss": 2.6422, + "theoretical_loss": 3.8478501113426073, + "tokens_seen": 588251136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041501504513540623, + "loss": 2.5214, + "theoretical_loss": 3.847805227363743, + "tokens_seen": 588316672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150050150451354, + "loss": 2.6265, + "theoretical_loss": 3.8477603497842763, + "tokens_seen": 588382208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149949849548646, + "loss": 2.6646, + "theoretical_loss": 3.8477154786025825, + "tokens_seen": 588447744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498495486459383, + "loss": 2.7582, + "theoretical_loss": 3.847670613817038, + "tokens_seen": 588513280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041497492477432296, + "loss": 2.8478, + "theoretical_loss": 3.847625755426018, + "tokens_seen": 588578816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149648946840522, + "loss": 2.6564, + "theoretical_loss": 3.8475809034278994, + "tokens_seen": 588644352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149548645937813, + "loss": 2.4809, + "theoretical_loss": 3.8475360578210602, + "tokens_seen": 588709888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041494483450351055, + "loss": 2.8239, + "theoretical_loss": 3.847491218603878, + "tokens_seen": 588775424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041493480441323974, + "loss": 2.6325, + "theoretical_loss": 3.847446385774731, + "tokens_seen": 588840960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149247743229689, + "loss": 2.8157, + "theoretical_loss": 3.847401559331998, + "tokens_seen": 588906496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149147442326981, + "loss": 2.8234, + "theoretical_loss": 3.8473567392740593, + "tokens_seen": 588972032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041490471414242733, + "loss": 2.6742, + "theoretical_loss": 3.8473119255992954, + "tokens_seen": 589037568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041489468405215646, + "loss": 2.9159, + "theoretical_loss": 3.8472671183060867, + "tokens_seen": 589103104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148846539618857, + "loss": 2.8963, + "theoretical_loss": 3.8472223173928146, + "tokens_seen": 589168640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148746238716148, + "loss": 2.7451, + "theoretical_loss": 3.8471775228578617, + "tokens_seen": 589234176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041486459378134406, + "loss": 2.7525, + "theoretical_loss": 3.8471327346996107, + "tokens_seen": 589299712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041485456369107324, + "loss": 2.73, + "theoretical_loss": 3.8470879529164437, + "tokens_seen": 589365248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148445336008024, + "loss": 2.8554, + "theoretical_loss": 3.847043177506746, + "tokens_seen": 589430784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148345035105316, + "loss": 2.9111, + "theoretical_loss": 3.8469984084689015, + "tokens_seen": 589496320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148244734202608, + "loss": 2.7169, + "theoretical_loss": 3.8469536458012943, + "tokens_seen": 589561856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041481444332998996, + "loss": 2.8186, + "theoretical_loss": 3.8469088895023114, + "tokens_seen": 589627392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148044132397192, + "loss": 2.8273, + "theoretical_loss": 3.8468641395703385, + "tokens_seen": 589692928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147943831494483, + "loss": 2.6757, + "theoretical_loss": 3.8468193960037618, + "tokens_seen": 589758464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 334849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.178940534591675, + "objective/train/theoretical_loss": 3.8467746588009692, + "objective/train/tokens_used": 610284000, + "theoretical_loss": 3.8467746588009692, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041478435305917756, + "loss": 2.6907, + "theoretical_loss": 3.8467746588009692, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147743229689067, + "loss": 2.798, + "theoretical_loss": 3.846729927960349, + "tokens_seen": 589889536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147642928786359, + "loss": 2.9737, + "theoretical_loss": 3.846685203480289, + "tokens_seen": 589955072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147542627883651, + "loss": 2.6387, + "theoretical_loss": 3.8466404853591785, + "tokens_seen": 590020608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147442326980943, + "loss": 2.5792, + "theoretical_loss": 3.8465957735954075, + "tokens_seen": 590086144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041473420260782347, + "loss": 3.0323, + "theoretical_loss": 3.8465510681873667, + "tokens_seen": 590151680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147241725175527, + "loss": 2.8036, + "theoretical_loss": 3.8465063691334453, + "tokens_seen": 590217216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041471414242728183, + "loss": 2.8123, + "theoretical_loss": 3.846461676432037, + "tokens_seen": 590282752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041470411233701106, + "loss": 2.9491, + "theoretical_loss": 3.8464169900815324, + "tokens_seen": 590348288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146940822467402, + "loss": 2.687, + "theoretical_loss": 3.8463723100803247, + "tokens_seen": 590413824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146840521564694, + "loss": 2.708, + "theoretical_loss": 3.846327636426807, + "tokens_seen": 590479360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146740220661986, + "loss": 2.7274, + "theoretical_loss": 3.8462829691193727, + "tokens_seen": 590544896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146639919759278, + "loss": 2.809, + "theoretical_loss": 3.8462383081564173, + "tokens_seen": 590610432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041465396188565697, + "loss": 2.8318, + "theoretical_loss": 3.846193653536335, + "tokens_seen": 590675968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041464393179538615, + "loss": 2.8892, + "theoretical_loss": 3.846149005257521, + "tokens_seen": 590741504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041463390170511533, + "loss": 2.6609, + "theoretical_loss": 3.8461043633183722, + "tokens_seen": 590807040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041462387161484457, + "loss": 2.5279, + "theoretical_loss": 3.8460597277172845, + "tokens_seen": 590872576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146138415245737, + "loss": 2.7456, + "theoretical_loss": 3.846015098452656, + "tokens_seen": 590938112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041460381143430293, + "loss": 2.9341, + "theoretical_loss": 3.8459704755228854, + "tokens_seen": 591003648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145937813440321, + "loss": 2.9464, + "theoretical_loss": 3.8459258589263694, + "tokens_seen": 591069184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145837512537613, + "loss": 2.7979, + "theoretical_loss": 3.8458812486615077, + "tokens_seen": 591134720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145737211634905, + "loss": 2.7745, + "theoretical_loss": 3.8458366447267003, + "tokens_seen": 591200256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041456369107321965, + "loss": 2.7959, + "theoretical_loss": 3.845792047120347, + "tokens_seen": 591265792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041455366098294884, + "loss": 2.8272, + "theoretical_loss": 3.845747455840849, + "tokens_seen": 591331328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454363089267807, + "loss": 2.9054, + "theoretical_loss": 3.8457028708866074, + "tokens_seen": 591396864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 335803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.751352071762085, + "objective/train/theoretical_loss": 3.845658292256025, + "objective/train/tokens_used": 611922400, + "theoretical_loss": 3.845658292256025, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145336008024072, + "loss": 2.7783, + "theoretical_loss": 3.845658292256025, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041452357071213643, + "loss": 2.7238, + "theoretical_loss": 3.8456137199475027, + "tokens_seen": 591527936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041451354062186556, + "loss": 2.6259, + "theoretical_loss": 3.845569153959445, + "tokens_seen": 591593472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145035105315948, + "loss": 2.8755, + "theoretical_loss": 3.8455245942902554, + "tokens_seen": 591659008 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414493480441324, + "loss": 2.9484, + "theoretical_loss": 3.8454800409383374, + "tokens_seen": 591724544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041448345035105316, + "loss": 2.7721, + "theoretical_loss": 3.845435493902097, + "tokens_seen": 591790080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041447342026078234, + "loss": 2.8939, + "theoretical_loss": 3.8453909531799395, + "tokens_seen": 591855616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144633901705115, + "loss": 2.9146, + "theoretical_loss": 3.84534641877027, + "tokens_seen": 591921152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144533600802407, + "loss": 2.7202, + "theoretical_loss": 3.845301890671496, + "tokens_seen": 591986688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041444332998996994, + "loss": 2.8713, + "theoretical_loss": 3.845257368882024, + "tokens_seen": 592052224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041443329989969906, + "loss": 3.1036, + "theoretical_loss": 3.8452128534002625, + "tokens_seen": 592117760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144232698094283, + "loss": 2.7427, + "theoretical_loss": 3.8451683442246196, + "tokens_seen": 592183296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041441323971915753, + "loss": 2.8624, + "theoretical_loss": 3.8451238413535043, + "tokens_seen": 592248832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041440320962888666, + "loss": 2.5677, + "theoretical_loss": 3.8450793447853258, + "tokens_seen": 592314368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143931795386159, + "loss": 2.8878, + "theoretical_loss": 3.8450348545184942, + "tokens_seen": 592379904 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414383149448345, + "loss": 2.5672, + "theoretical_loss": 3.84499037055142, + "tokens_seen": 592445440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041437311935807426, + "loss": 2.7789, + "theoretical_loss": 3.8449458928825155, + "tokens_seen": 592510976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041436308926780344, + "loss": 2.9836, + "theoretical_loss": 3.8449014215101913, + "tokens_seen": 592576512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143530591775326, + "loss": 2.8149, + "theoretical_loss": 3.84485695643286, + "tokens_seen": 592642048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143430290872618, + "loss": 2.8165, + "theoretical_loss": 3.8448124976489346, + "tokens_seen": 592707584 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414332998996991, + "loss": 3.0223, + "theoretical_loss": 3.8447680451568296, + "tokens_seen": 592773120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041432296890672016, + "loss": 2.7377, + "theoretical_loss": 3.844723598954958, + "tokens_seen": 592838656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143129388164494, + "loss": 2.8208, + "theoretical_loss": 3.8446791590417346, + "tokens_seen": 592904192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143029087261785, + "loss": 2.7563, + "theoretical_loss": 3.8446347254155753, + "tokens_seen": 592969728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041429287863590776, + "loss": 2.9717, + "theoretical_loss": 3.8445902980748956, + "tokens_seen": 593035264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 336288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.919393539428711, + "objective/train/theoretical_loss": 3.8445458770181116, + "objective/train/tokens_used": 613560800, + "theoretical_loss": 3.8445458770181116, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142828485456369, + "loss": 2.8637, + "theoretical_loss": 3.8445458770181116, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142728184553661, + "loss": 2.6485, + "theoretical_loss": 3.844501462243641, + "tokens_seen": 593166336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142627883650953, + "loss": 2.8594, + "theoretical_loss": 3.8444570537499, + "tokens_seen": 593231872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142527582748245, + "loss": 2.4827, + "theoretical_loss": 3.8444126515353076, + "tokens_seen": 593297408 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041424272818455367, + "loss": 2.7269, + "theoretical_loss": 3.8443682555982837, + "tokens_seen": 593362944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142326980942829, + "loss": 2.8108, + "theoretical_loss": 3.8443238659372456, + "tokens_seen": 593428480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041422266800401203, + "loss": 2.8032, + "theoretical_loss": 3.844279482550614, + "tokens_seen": 593494016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041421263791374126, + "loss": 2.8399, + "theoretical_loss": 3.84423510543681, + "tokens_seen": 593559552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142026078234704, + "loss": 2.7256, + "theoretical_loss": 3.8441907345942528, + "tokens_seen": 593625088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141925777331996, + "loss": 2.891, + "theoretical_loss": 3.844146370021366, + "tokens_seen": 593690624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141825476429288, + "loss": 2.8849, + "theoretical_loss": 3.8441020117165703, + "tokens_seen": 593756160 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414172517552658, + "loss": 2.8343, + "theoretical_loss": 3.8440576596782887, + "tokens_seen": 593821696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041416248746238717, + "loss": 2.9734, + "theoretical_loss": 3.844013313904945, + "tokens_seen": 593887232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041415245737211635, + "loss": 2.7278, + "theoretical_loss": 3.8439689743949628, + "tokens_seen": 593952768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041414242728184553, + "loss": 2.6113, + "theoretical_loss": 3.8439246411467662, + "tokens_seen": 594018304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041413239719157477, + "loss": 2.7153, + "theoretical_loss": 3.8438803141587807, + "tokens_seen": 594083840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141223671013039, + "loss": 2.9235, + "theoretical_loss": 3.8438359934294315, + "tokens_seen": 594149376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041411233701103313, + "loss": 2.7539, + "theoretical_loss": 3.843791678957145, + "tokens_seen": 594214912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141023069207623, + "loss": 2.8045, + "theoretical_loss": 3.8437473707403482, + "tokens_seen": 594280448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140922768304915, + "loss": 2.9262, + "theoretical_loss": 3.843703068777467, + "tokens_seen": 594345984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140822467402207, + "loss": 2.9616, + "theoretical_loss": 3.843658773066931, + "tokens_seen": 594411520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041407221664994985, + "loss": 2.8237, + "theoretical_loss": 3.8436144836071673, + "tokens_seen": 594477056 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041406218655967904, + "loss": 2.9555, + "theoretical_loss": 3.8435702003966052, + "tokens_seen": 594542592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041405215646940827, + "loss": 2.7086, + "theoretical_loss": 3.843525923433675, + "tokens_seen": 594608128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140421263791374, + "loss": 2.8386, + "theoretical_loss": 3.8434816527168056, + "tokens_seen": 594673664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 337523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4961068630218506, + "objective/train/theoretical_loss": 3.8434373882444284, + "objective/train/tokens_used": 615199200, + "theoretical_loss": 3.8434373882444284, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041403209628886663, + "loss": 2.7317, + "theoretical_loss": 3.8434373882444284, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041402206619859576, + "loss": 2.6545, + "theoretical_loss": 3.8433931300149746, + "tokens_seen": 594804736 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414012036108325, + "loss": 2.952, + "theoretical_loss": 3.843348878026876, + "tokens_seen": 594870272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140020060180542, + "loss": 2.8919, + "theoretical_loss": 3.843304632278565, + "tokens_seen": 594935808 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041399197592778336, + "loss": 2.9386, + "theoretical_loss": 3.8432603927684745, + "tokens_seen": 595001344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041398194583751254, + "loss": 2.533, + "theoretical_loss": 3.8432161594950376, + "tokens_seen": 595066880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139719157472417, + "loss": 2.7428, + "theoretical_loss": 3.8431719324566895, + "tokens_seen": 595132416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139618856569709, + "loss": 2.6911, + "theoretical_loss": 3.8431277116518636, + "tokens_seen": 595197952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041395185556670014, + "loss": 2.8322, + "theoretical_loss": 3.8430834970789953, + "tokens_seen": 595263488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041394182547642926, + "loss": 2.8112, + "theoretical_loss": 3.843039288736521, + "tokens_seen": 595329024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139317953861585, + "loss": 2.7172, + "theoretical_loss": 3.842995086622877, + "tokens_seen": 595394560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139217652958877, + "loss": 2.713, + "theoretical_loss": 3.8429508907364998, + "tokens_seen": 595460096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041391173520561686, + "loss": 2.8304, + "theoretical_loss": 3.842906701075827, + "tokens_seen": 595525632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041390170511534604, + "loss": 2.942, + "theoretical_loss": 3.8428625176392965, + "tokens_seen": 595591168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138916750250752, + "loss": 2.8326, + "theoretical_loss": 3.8428183404253473, + "tokens_seen": 595656704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138816449348044, + "loss": 2.8756, + "theoretical_loss": 3.8427741694324182, + "tokens_seen": 595722240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041387161484453364, + "loss": 2.6363, + "theoretical_loss": 3.8427300046589488, + "tokens_seen": 595787776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041386158475426277, + "loss": 2.7768, + "theoretical_loss": 3.8426858461033797, + "tokens_seen": 595853312 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413851554663992, + "loss": 2.6984, + "theoretical_loss": 3.8426416937641514, + "tokens_seen": 595918848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041384152457372113, + "loss": 2.5968, + "theoretical_loss": 3.842597547639706, + "tokens_seen": 595984384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041383149448345036, + "loss": 2.9943, + "theoretical_loss": 3.842553407728485, + "tokens_seen": 596049920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041382146439317954, + "loss": 2.6488, + "theoretical_loss": 3.842509274028931, + "tokens_seen": 596115456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138114343029087, + "loss": 2.6736, + "theoretical_loss": 3.8424651465394866, + "tokens_seen": 596180992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138014042126379, + "loss": 2.6599, + "theoretical_loss": 3.842421025258596, + "tokens_seen": 596246528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137913741223671, + "loss": 2.826, + "theoretical_loss": 3.8423769101847034, + "tokens_seen": 596312064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 338188, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9216253757476807, + "objective/train/theoretical_loss": 3.842332801316254, + "objective/train/tokens_used": 616837600, + "theoretical_loss": 3.842332801316254, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041378134403209627, + "loss": 2.8479, + "theoretical_loss": 3.842332801316254, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137713139418255, + "loss": 2.7266, + "theoretical_loss": 3.842288698651692, + "tokens_seen": 596443136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041376128385155463, + "loss": 2.689, + "theoretical_loss": 3.8422446021894645, + "tokens_seen": 596508672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041375125376128387, + "loss": 2.9293, + "theoretical_loss": 3.842200511928017, + "tokens_seen": 596574208 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041374122367101305, + "loss": 2.5976, + "theoretical_loss": 3.842156427865797, + "tokens_seen": 596639744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041373119358074223, + "loss": 2.9682, + "theoretical_loss": 3.842112350001252, + "tokens_seen": 596705280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137211634904714, + "loss": 2.8541, + "theoretical_loss": 3.84206827833283, + "tokens_seen": 596770816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137111334002006, + "loss": 2.7331, + "theoretical_loss": 3.84202421285898, + "tokens_seen": 596836352 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041370110330992977, + "loss": 2.9806, + "theoretical_loss": 3.8419801535781515, + "tokens_seen": 596901888 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413691073219659, + "loss": 2.6249, + "theoretical_loss": 3.841936100488793, + "tokens_seen": 596967424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041368104312938813, + "loss": 2.7589, + "theoretical_loss": 3.8418920535893566, + "tokens_seen": 597032960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041367101303911737, + "loss": 2.6685, + "theoretical_loss": 3.841848012878292, + "tokens_seen": 597098496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041366098294884655, + "loss": 2.8725, + "theoretical_loss": 3.8418039783540507, + "tokens_seen": 597164032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041365095285857573, + "loss": 2.767, + "theoretical_loss": 3.841759950015086, + "tokens_seen": 597229568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041364092276830497, + "loss": 2.6848, + "theoretical_loss": 3.841715927859849, + "tokens_seen": 597295104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136308926780341, + "loss": 2.7853, + "theoretical_loss": 3.8416719118867944, + "tokens_seen": 597360640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041362086258776333, + "loss": 2.6599, + "theoretical_loss": 3.841627902094374, + "tokens_seen": 597426176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136108324974925, + "loss": 2.919, + "theoretical_loss": 3.8415838984810433, + "tokens_seen": 597491712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136008024072217, + "loss": 2.8007, + "theoretical_loss": 3.841539901045257, + "tokens_seen": 597557248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135907723169509, + "loss": 2.7802, + "theoretical_loss": 3.841495909785471, + "tokens_seen": 597622784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041358074222668005, + "loss": 2.7021, + "theoretical_loss": 3.8414519247001397, + "tokens_seen": 597688320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041357071213640924, + "loss": 2.6627, + "theoretical_loss": 3.8414079457877213, + "tokens_seen": 597753856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041356068204613847, + "loss": 2.7435, + "theoretical_loss": 3.8413639730466715, + "tokens_seen": 597819392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135506519558676, + "loss": 2.9956, + "theoretical_loss": 3.8413200064754487, + "tokens_seen": 597884928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041354062186559683, + "loss": 2.6928, + "theoretical_loss": 3.841276046072511, + "tokens_seen": 597950464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 339419, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.796257972717285, + "objective/train/theoretical_loss": 3.841232091836317, + "objective/train/tokens_used": 618476000, + "theoretical_loss": 3.841232091836317, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041353059177532596, + "loss": 2.649, + "theoretical_loss": 3.841232091836317, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135205616850552, + "loss": 2.9916, + "theoretical_loss": 3.8411881437653252, + "tokens_seen": 598081536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135105315947844, + "loss": 2.767, + "theoretical_loss": 3.841144201857997, + "tokens_seen": 598147072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041350050150451356, + "loss": 2.6961, + "theoretical_loss": 3.8411002661127913, + "tokens_seen": 598212608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041349047141424274, + "loss": 2.5988, + "theoretical_loss": 3.8410563365281702, + "tokens_seen": 598278144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134804413239719, + "loss": 2.4411, + "theoretical_loss": 3.8410124131025944, + "tokens_seen": 598343680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134704112337011, + "loss": 2.9326, + "theoretical_loss": 3.8409684958345265, + "tokens_seen": 598409216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041346038114343034, + "loss": 2.7858, + "theoretical_loss": 3.840924584722428, + "tokens_seen": 598474752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041345035105315946, + "loss": 2.6159, + "theoretical_loss": 3.840880679764763, + "tokens_seen": 598540288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134403209628887, + "loss": 2.9933, + "theoretical_loss": 3.8408367809599957, + "tokens_seen": 598605824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134302908726179, + "loss": 2.634, + "theoretical_loss": 3.8407928883065887, + "tokens_seen": 598671360 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041342026078234706, + "loss": 2.5301, + "theoretical_loss": 3.840749001803008, + "tokens_seen": 598736896 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041341023069207624, + "loss": 2.6363, + "theoretical_loss": 3.8407051214477193, + "tokens_seen": 598802432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134002006018054, + "loss": 2.7874, + "theoretical_loss": 3.8406612472391872, + "tokens_seen": 598867968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133901705115346, + "loss": 2.9248, + "theoretical_loss": 3.8406173791758786, + "tokens_seen": 598933504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041338014042126384, + "loss": 2.8438, + "theoretical_loss": 3.840573517256261, + "tokens_seen": 598999040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041337011033099297, + "loss": 2.9863, + "theoretical_loss": 3.8405296614788016, + "tokens_seen": 599064576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133600802407222, + "loss": 2.4437, + "theoretical_loss": 3.8404858118419685, + "tokens_seen": 599130112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041335005015045133, + "loss": 2.6434, + "theoretical_loss": 3.84044196834423, + "tokens_seen": 599195648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041334002006018056, + "loss": 2.7596, + "theoretical_loss": 3.840398130984056, + "tokens_seen": 599261184 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041332998996990975, + "loss": 2.8734, + "theoretical_loss": 3.840354299759916, + "tokens_seen": 599326720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133199598796389, + "loss": 2.8495, + "theoretical_loss": 3.84031047467028, + "tokens_seen": 599392256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133099297893681, + "loss": 2.6736, + "theoretical_loss": 3.840266655713619, + "tokens_seen": 599457792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132998996990973, + "loss": 3.0788, + "theoretical_loss": 3.840222842888405, + "tokens_seen": 599523328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041328986960882647, + "loss": 3.0841, + "theoretical_loss": 3.840179036193109, + "tokens_seen": 599588864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 340168, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.811565399169922, + "objective/train/theoretical_loss": 3.840135235626204, + "objective/train/tokens_used": 620114400, + "theoretical_loss": 3.840135235626204, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132798395185557, + "loss": 2.6611, + "theoretical_loss": 3.840135235626204, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041326980942828483, + "loss": 2.7923, + "theoretical_loss": 3.8400914411861624, + "tokens_seen": 599719936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041325977933801407, + "loss": 2.8316, + "theoretical_loss": 3.8400476528714584, + "tokens_seen": 599785472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041324974924774325, + "loss": 2.8943, + "theoretical_loss": 3.840003870680566, + "tokens_seen": 599851008 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041323971915747243, + "loss": 2.6886, + "theoretical_loss": 3.8399600946119605, + "tokens_seen": 599916544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132296890672016, + "loss": 2.8353, + "theoretical_loss": 3.8399163246641157, + "tokens_seen": 599982080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132196589769308, + "loss": 3.022, + "theoretical_loss": 3.839872560835509, + "tokens_seen": 600047616 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041320962888665997, + "loss": 2.9232, + "theoretical_loss": 3.839828803124615, + "tokens_seen": 600113152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131995987963892, + "loss": 3.0362, + "theoretical_loss": 3.839785051529912, + "tokens_seen": 600178688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041318956870611833, + "loss": 2.633, + "theoretical_loss": 3.839741306049876, + "tokens_seen": 600244224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041317953861584757, + "loss": 2.9153, + "theoretical_loss": 3.839697566682987, + "tokens_seen": 600309760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131695085255767, + "loss": 2.8834, + "theoretical_loss": 3.8396538334277217, + "tokens_seen": 600375296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041315947843530593, + "loss": 2.8856, + "theoretical_loss": 3.839610106282559, + "tokens_seen": 600440832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131494483450351, + "loss": 2.9453, + "theoretical_loss": 3.83956638524598, + "tokens_seen": 600506368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131394182547643, + "loss": 2.6424, + "theoretical_loss": 3.8395226703164638, + "tokens_seen": 600571904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131293881644935, + "loss": 3.0177, + "theoretical_loss": 3.8394789614924916, + "tokens_seen": 600637440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131193580742227, + "loss": 2.7729, + "theoretical_loss": 3.8394352587725438, + "tokens_seen": 600702976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041310932798395184, + "loss": 2.9922, + "theoretical_loss": 3.839391562155103, + "tokens_seen": 600768512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130992978936811, + "loss": 2.7626, + "theoretical_loss": 3.8393478716386507, + "tokens_seen": 600834048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130892678034102, + "loss": 2.7941, + "theoretical_loss": 3.8393041872216704, + "tokens_seen": 600899584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041307923771313944, + "loss": 2.4599, + "theoretical_loss": 3.8392605089026457, + "tokens_seen": 600965120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130692076228686, + "loss": 2.6901, + "theoretical_loss": 3.83921683668006, + "tokens_seen": 601030656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130591775325978, + "loss": 2.7902, + "theoretical_loss": 3.8391731705523977, + "tokens_seen": 601096192 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413049147442327, + "loss": 2.6181, + "theoretical_loss": 3.8391295105181436, + "tokens_seen": 601161728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041303911735205616, + "loss": 2.7678, + "theoretical_loss": 3.8390858565757844, + "tokens_seen": 601227264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 341333, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.842754364013672, + "objective/train/theoretical_loss": 3.8390422087238045, + "objective/train/tokens_used": 621752800, + "theoretical_loss": 3.8390422087238045, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041302908726178534, + "loss": 2.7321, + "theoretical_loss": 3.8390422087238045, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130190571715146, + "loss": 2.9798, + "theoretical_loss": 3.8389985669606927, + "tokens_seen": 601358336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130090270812437, + "loss": 2.5315, + "theoretical_loss": 3.8389549312849343, + "tokens_seen": 601423872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041299899699097294, + "loss": 2.8848, + "theoretical_loss": 3.838911301695018, + "tokens_seen": 601489408 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041298896690070207, + "loss": 2.8289, + "theoretical_loss": 3.8388676781894313, + "tokens_seen": 601554944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129789368104313, + "loss": 2.8317, + "theoretical_loss": 3.8388240607666635, + "tokens_seen": 601620480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129689067201605, + "loss": 2.7689, + "theoretical_loss": 3.838780449425204, + "tokens_seen": 601686016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041295887662988966, + "loss": 2.5929, + "theoretical_loss": 3.8387368441635425, + "tokens_seen": 601751552 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041294884653961884, + "loss": 2.9393, + "theoretical_loss": 3.8386932449801696, + "tokens_seen": 601817088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129388164493481, + "loss": 2.8339, + "theoretical_loss": 3.8386496518735767, + "tokens_seen": 601882624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129287863590772, + "loss": 2.7379, + "theoretical_loss": 3.8386060648422538, + "tokens_seen": 601948160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041291875626880644, + "loss": 2.9387, + "theoretical_loss": 3.8385624838846946, + "tokens_seen": 602013696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129087261785356, + "loss": 2.8846, + "theoretical_loss": 3.838518908999391, + "tokens_seen": 602079232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128986960882648, + "loss": 2.9549, + "theoretical_loss": 3.8384753401848353, + "tokens_seen": 602144768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041288866599799404, + "loss": 2.8541, + "theoretical_loss": 3.8384317774395225, + "tokens_seen": 602210304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041287863590772317, + "loss": 2.7912, + "theoretical_loss": 3.8383882207619457, + "tokens_seen": 602275840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128686058174524, + "loss": 2.635, + "theoretical_loss": 3.838344670150601, + "tokens_seen": 602341376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041285857572718153, + "loss": 2.9703, + "theoretical_loss": 3.838301125603982, + "tokens_seen": 602406912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041284854563691076, + "loss": 2.989, + "theoretical_loss": 3.8382575871205855, + "tokens_seen": 602472448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041283851554663995, + "loss": 2.6135, + "theoretical_loss": 3.8382140546989074, + "tokens_seen": 602537984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128284854563691, + "loss": 2.7771, + "theoretical_loss": 3.8381705283374457, + "tokens_seen": 602603520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128184553660983, + "loss": 2.6952, + "theoretical_loss": 3.8381270080346956, + "tokens_seen": 602669056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128084252758275, + "loss": 2.5534, + "theoretical_loss": 3.8380834937891573, + "tokens_seen": 602734592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041279839518555667, + "loss": 2.8246, + "theoretical_loss": 3.8380399855993286, + "tokens_seen": 602800128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127883650952859, + "loss": 2.5559, + "theoretical_loss": 3.8379964834637073, + "tokens_seen": 602865664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 342479, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2375457286834717, + "objective/train/theoretical_loss": 3.8379529873807945, + "objective/train/tokens_used": 623391200, + "theoretical_loss": 3.8379529873807945, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041277833500501503, + "loss": 2.7622, + "theoretical_loss": 3.8379529873807945, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041276830491474427, + "loss": 2.7673, + "theoretical_loss": 3.8379094973490897, + "tokens_seen": 602996736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041275827482447345, + "loss": 3.0164, + "theoretical_loss": 3.837866013367093, + "tokens_seen": 603062272 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041274824473420263, + "loss": 2.8215, + "theoretical_loss": 3.8378225354333066, + "tokens_seen": 603127808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127382146439318, + "loss": 2.6788, + "theoretical_loss": 3.8377790635462308, + "tokens_seen": 603193344 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412728184553661, + "loss": 2.8211, + "theoretical_loss": 3.83773559770437, + "tokens_seen": 603258880 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041271815446339017, + "loss": 2.9036, + "theoretical_loss": 3.8376921379062248, + "tokens_seen": 603324416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127081243731194, + "loss": 2.5925, + "theoretical_loss": 3.8376486841502992, + "tokens_seen": 603389952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041269809428284854, + "loss": 2.7658, + "theoretical_loss": 3.837605236435097, + "tokens_seen": 603455488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041268806419257777, + "loss": 2.5781, + "theoretical_loss": 3.8375617947591234, + "tokens_seen": 603521024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126780341023069, + "loss": 2.856, + "theoretical_loss": 3.8375183591208826, + "tokens_seen": 603586560 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041266800401203613, + "loss": 2.5308, + "theoretical_loss": 3.837474929518879, + "tokens_seen": 603652096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126579739217653, + "loss": 2.6879, + "theoretical_loss": 3.8374315059516206, + "tokens_seen": 603717632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126479438314945, + "loss": 2.8421, + "theoretical_loss": 3.8373880884176126, + "tokens_seen": 603783168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126379137412237, + "loss": 2.881, + "theoretical_loss": 3.837344676915362, + "tokens_seen": 603848704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126278836509529, + "loss": 2.3681, + "theoretical_loss": 3.8373012714433767, + "tokens_seen": 603914240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041261785356068204, + "loss": 2.6518, + "theoretical_loss": 3.8372578720001647, + "tokens_seen": 603979776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126078234704113, + "loss": 2.8608, + "theoretical_loss": 3.8372144785842344, + "tokens_seen": 604045312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125977933801404, + "loss": 2.5857, + "theoretical_loss": 3.8371710911940955, + "tokens_seen": 604110848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041258776328986964, + "loss": 2.3453, + "theoretical_loss": 3.8371277098282572, + "tokens_seen": 604176384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125777331995988, + "loss": 2.8187, + "theoretical_loss": 3.8370843344852297, + "tokens_seen": 604241920 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412567703109328, + "loss": 2.6687, + "theoretical_loss": 3.8370409651635238, + "tokens_seen": 604307456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125576730190572, + "loss": 2.7338, + "theoretical_loss": 3.8369976018616505, + "tokens_seen": 604372992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041254764292878636, + "loss": 2.8385, + "theoretical_loss": 3.8369542445781226, + "tokens_seen": 604438528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041253761283851554, + "loss": 2.6823, + "theoretical_loss": 3.836910893311451, + "tokens_seen": 604504064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 342997, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.627711534500122, + "objective/train/theoretical_loss": 3.83686754806015, + "objective/train/tokens_used": 625029600, + "theoretical_loss": 3.83686754806015, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125275827482448, + "loss": 2.6497, + "theoretical_loss": 3.83686754806015, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125175526579739, + "loss": 2.9453, + "theoretical_loss": 3.8368242088227316, + "tokens_seen": 604635136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041250752256770314, + "loss": 2.9436, + "theoretical_loss": 3.836780875597711, + "tokens_seen": 604700672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041249749247743227, + "loss": 2.6017, + "theoretical_loss": 3.836737548383601, + "tokens_seen": 604766208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124874623871615, + "loss": 2.9184, + "theoretical_loss": 3.836694227178919, + "tokens_seen": 604831744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124774322968907, + "loss": 2.8143, + "theoretical_loss": 3.8366509119821774, + "tokens_seen": 604897280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041246740220661986, + "loss": 2.9792, + "theoretical_loss": 3.836607602791895, + "tokens_seen": 604962816 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041245737211634904, + "loss": 2.7049, + "theoretical_loss": 3.8365642996065867, + "tokens_seen": 605028352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124473420260783, + "loss": 2.4616, + "theoretical_loss": 3.83652100242477, + "tokens_seen": 605093888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124373119358074, + "loss": 2.7036, + "theoretical_loss": 3.836477711244963, + "tokens_seen": 605159424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041242728184553664, + "loss": 2.7757, + "theoretical_loss": 3.836434426065683, + "tokens_seen": 605224960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041241725175526577, + "loss": 2.8633, + "theoretical_loss": 3.836391146885449, + "tokens_seen": 605290496 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412407221664995, + "loss": 2.8372, + "theoretical_loss": 3.8363478737027803, + "tokens_seen": 605356032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123971915747242, + "loss": 2.6962, + "theoretical_loss": 3.836304606516197, + "tokens_seen": 605421568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041238716148445337, + "loss": 2.8052, + "theoretical_loss": 3.8362613453242185, + "tokens_seen": 605487104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041237713139418255, + "loss": 2.6351, + "theoretical_loss": 3.8362180901253655, + "tokens_seen": 605552640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041236710130391173, + "loss": 2.7867, + "theoretical_loss": 3.83617484091816, + "tokens_seen": 605618176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123570712136409, + "loss": 2.6174, + "theoretical_loss": 3.8361315977011237, + "tokens_seen": 605683712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041234704112337015, + "loss": 2.62, + "theoretical_loss": 3.836088360472779, + "tokens_seen": 605749248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041233701103309927, + "loss": 2.7599, + "theoretical_loss": 3.8360451292316475, + "tokens_seen": 605814784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123269809428285, + "loss": 2.6258, + "theoretical_loss": 3.8360019039762543, + "tokens_seen": 605880320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041231695085255763, + "loss": 2.8677, + "theoretical_loss": 3.835958684705122, + "tokens_seen": 605945856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041230692076228687, + "loss": 2.9551, + "theoretical_loss": 3.835915471416776, + "tokens_seen": 606011392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041229689067201605, + "loss": 2.7614, + "theoretical_loss": 3.8358722641097405, + "tokens_seen": 606076928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041228686058174523, + "loss": 2.7635, + "theoretical_loss": 3.8358290627825413, + "tokens_seen": 606142464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 344327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1000027656555176, + "objective/train/theoretical_loss": 3.835785867433705, + "objective/train/tokens_used": 626668000, + "theoretical_loss": 3.835785867433705, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122768304914744, + "loss": 2.8939, + "theoretical_loss": 3.835785867433705, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041226680040120365, + "loss": 2.8328, + "theoretical_loss": 3.8357426780617567, + "tokens_seen": 606273536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122567703109328, + "loss": 2.9931, + "theoretical_loss": 3.8356994946652248, + "tokens_seen": 606339072 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412246740220662, + "loss": 2.5709, + "theoretical_loss": 3.835656317242636, + "tokens_seen": 606404608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041223671013039114, + "loss": 2.7535, + "theoretical_loss": 3.835613145792519, + "tokens_seen": 606470144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122266800401204, + "loss": 2.6616, + "theoretical_loss": 3.8355699803134016, + "tokens_seen": 606535680 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041221664994984955, + "loss": 2.766, + "theoretical_loss": 3.8355268208038136, + "tokens_seen": 606601216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041220661985957874, + "loss": 2.7006, + "theoretical_loss": 3.8354836672622845, + "tokens_seen": 606666752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121965897693079, + "loss": 2.8494, + "theoretical_loss": 3.835440519687345, + "tokens_seen": 606732288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121865596790371, + "loss": 2.797, + "theoretical_loss": 3.8353973780775243, + "tokens_seen": 606797824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121765295887663, + "loss": 2.8522, + "theoretical_loss": 3.8353542424313547, + "tokens_seen": 606863360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121664994984955, + "loss": 2.9709, + "theoretical_loss": 3.835311112747368, + "tokens_seen": 606928896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121564694082247, + "loss": 2.7044, + "theoretical_loss": 3.8352679890240964, + "tokens_seen": 606994432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121464393179539, + "loss": 2.9497, + "theoretical_loss": 3.8352248712600723, + "tokens_seen": 607059968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121364092276831, + "loss": 2.8229, + "theoretical_loss": 3.835181759453829, + "tokens_seen": 607125504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041212637913741224, + "loss": 2.7877, + "theoretical_loss": 3.8351386536039005, + "tokens_seen": 607191040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121163490471415, + "loss": 2.8895, + "theoretical_loss": 3.8350955537088214, + "tokens_seen": 607256576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121063189568706, + "loss": 2.7407, + "theoretical_loss": 3.835052459767126, + "tokens_seen": 607322112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041209628886659984, + "loss": 2.8907, + "theoretical_loss": 3.8350093717773497, + "tokens_seen": 607387648 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412086258776329, + "loss": 2.6943, + "theoretical_loss": 3.8349662897380288, + "tokens_seen": 607453184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120762286860582, + "loss": 2.8111, + "theoretical_loss": 3.8349232136476994, + "tokens_seen": 607518720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120661985957874, + "loss": 2.6154, + "theoretical_loss": 3.8348801435048987, + "tokens_seen": 607584256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041205616850551656, + "loss": 2.7947, + "theoretical_loss": 3.834837079308164, + "tokens_seen": 607649792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041204613841524574, + "loss": 2.4331, + "theoretical_loss": 3.834794021056033, + "tokens_seen": 607715328 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412036108324975, + "loss": 2.5009, + "theoretical_loss": 3.8347509687470445, + "tokens_seen": 607780864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 345027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9385602474212646, + "objective/train/theoretical_loss": 3.8347079223797373, + "objective/train/tokens_used": 628306400, + "theoretical_loss": 3.8347079223797373, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120260782347041, + "loss": 2.7218, + "theoretical_loss": 3.8347079223797373, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041201604814443334, + "loss": 2.4772, + "theoretical_loss": 3.8346648819526505, + "tokens_seen": 607911936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041200601805416247, + "loss": 2.6235, + "theoretical_loss": 3.834621847464325, + "tokens_seen": 607977472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119959879638917, + "loss": 2.7243, + "theoretical_loss": 3.834578818913301, + "tokens_seen": 608043008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119859578736209, + "loss": 3.0051, + "theoretical_loss": 3.8345357962981197, + "tokens_seen": 608108544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041197592778335006, + "loss": 2.7735, + "theoretical_loss": 3.834492779617322, + "tokens_seen": 608174080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041196589769307924, + "loss": 2.628, + "theoretical_loss": 3.834449768869451, + "tokens_seen": 608239616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119558676028085, + "loss": 2.6384, + "theoretical_loss": 3.834406764053048, + "tokens_seen": 608305152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119458375125376, + "loss": 2.5853, + "theoretical_loss": 3.834363765166657, + "tokens_seen": 608370688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041193580742226684, + "loss": 2.8413, + "theoretical_loss": 3.834320772208822, + "tokens_seen": 608436224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041192577733199597, + "loss": 2.8341, + "theoretical_loss": 3.8342777851780863, + "tokens_seen": 608501760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119157472417252, + "loss": 2.5505, + "theoretical_loss": 3.8342348040729943, + "tokens_seen": 608567296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119057171514544, + "loss": 2.6599, + "theoretical_loss": 3.8341918288920924, + "tokens_seen": 608632832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041189568706118357, + "loss": 2.7973, + "theoretical_loss": 3.8341488596339257, + "tokens_seen": 608698368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041188565697091275, + "loss": 2.799, + "theoretical_loss": 3.8341058962970402, + "tokens_seen": 608763904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041187562688064193, + "loss": 2.8924, + "theoretical_loss": 3.834062938879982, + "tokens_seen": 608829440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118655967903711, + "loss": 2.6892, + "theoretical_loss": 3.8340199873813, + "tokens_seen": 608894976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041185556670010035, + "loss": 2.8457, + "theoretical_loss": 3.8339770417995407, + "tokens_seen": 608960512 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041184553660982947, + "loss": 2.9086, + "theoretical_loss": 3.8339341021332527, + "tokens_seen": 609026048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118355065195587, + "loss": 2.7818, + "theoretical_loss": 3.833891168380984, + "tokens_seen": 609091584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041182547642928783, + "loss": 2.6026, + "theoretical_loss": 3.833848240541286, + "tokens_seen": 609157120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041181544633901707, + "loss": 2.7824, + "theoretical_loss": 3.833805318612706, + "tokens_seen": 609222656 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041180541624874625, + "loss": 2.6473, + "theoretical_loss": 3.8337624025937957, + "tokens_seen": 609288192 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041179538615847543, + "loss": 2.7189, + "theoretical_loss": 3.8337194924831053, + "tokens_seen": 609353728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117853560682046, + "loss": 2.7765, + "theoretical_loss": 3.833676588279187, + "tokens_seen": 609419264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 346111, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7798197269439697, + "objective/train/theoretical_loss": 3.8336336899805916, + "objective/train/tokens_used": 629944800, + "theoretical_loss": 3.8336336899805916, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041177532597793385, + "loss": 2.5951, + "theoretical_loss": 3.8336336899805916, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.000411765295887663, + "loss": 2.6398, + "theoretical_loss": 3.833590797585872, + "tokens_seen": 609550336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117552657973922, + "loss": 2.7884, + "theoretical_loss": 3.833547911093581, + "tokens_seen": 609615872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041174523570712134, + "loss": 2.7703, + "theoretical_loss": 3.8335050305022715, + "tokens_seen": 609681408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117352056168506, + "loss": 2.696, + "theoretical_loss": 3.833462155810498, + "tokens_seen": 609746944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041172517552657975, + "loss": 2.8528, + "theoretical_loss": 3.833419287016815, + "tokens_seen": 609812480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041171514543630894, + "loss": 2.6289, + "theoretical_loss": 3.8333764241197765, + "tokens_seen": 609878016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117051153460381, + "loss": 2.6666, + "theoretical_loss": 3.833333567117939, + "tokens_seen": 609943552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116950852557673, + "loss": 2.97, + "theoretical_loss": 3.8332907160098575, + "tokens_seen": 610009088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116850551654965, + "loss": 2.8593, + "theoretical_loss": 3.8332478707940885, + "tokens_seen": 610074624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116750250752257, + "loss": 2.6773, + "theoretical_loss": 3.8332050314691894, + "tokens_seen": 610140160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041166499498495484, + "loss": 2.8794, + "theoretical_loss": 3.833162198033718, + "tokens_seen": 610205696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116549648946841, + "loss": 2.9646, + "theoretical_loss": 3.833119370486231, + "tokens_seen": 610271232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116449348044132, + "loss": 2.7601, + "theoretical_loss": 3.8330765488252876, + "tokens_seen": 610336768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041163490471414244, + "loss": 2.6683, + "theoretical_loss": 3.8330337330494473, + "tokens_seen": 610402304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116248746238716, + "loss": 2.8002, + "theoretical_loss": 3.8329909231572685, + "tokens_seen": 610467840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004116148445336008, + "loss": 2.7484, + "theoretical_loss": 3.832948119147312, + "tokens_seen": 610533376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041160481444333, + "loss": 2.6991, + "theoretical_loss": 3.8329053210181376, + "tokens_seen": 610598912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115947843530592, + "loss": 2.7888, + "theoretical_loss": 3.8328625287683065, + "tokens_seen": 610664448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041158475426278834, + "loss": 2.8296, + "theoretical_loss": 3.8328197423963806, + "tokens_seen": 610729984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115747241725176, + "loss": 2.712, + "theoretical_loss": 3.8327769619009215, + "tokens_seen": 610795520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115646940822467, + "loss": 2.8347, + "theoretical_loss": 3.832734187280492, + "tokens_seen": 610861056 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041155466399197594, + "loss": 2.9096, + "theoretical_loss": 3.832691418533655, + "tokens_seen": 610926592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115446339017051, + "loss": 2.4744, + "theoretical_loss": 3.832648655658973, + "tokens_seen": 610992128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115346038114343, + "loss": 2.8143, + "theoretical_loss": 3.8326058986550127, + "tokens_seen": 611057664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 346814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.812352180480957, + "objective/train/theoretical_loss": 3.832563147520336, + "objective/train/tokens_used": 631583200, + "theoretical_loss": 3.832563147520336, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115245737211635, + "loss": 2.8388, + "theoretical_loss": 3.832563147520336, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041151454363089267, + "loss": 2.8298, + "theoretical_loss": 3.8325204022535084, + "tokens_seen": 611188736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041150451354062185, + "loss": 2.8495, + "theoretical_loss": 3.832477662853096, + "tokens_seen": 611254272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114944834503511, + "loss": 2.829, + "theoretical_loss": 3.8324349293176656, + "tokens_seen": 611319808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114844533600802, + "loss": 2.6977, + "theoretical_loss": 3.8323922016457823, + "tokens_seen": 611385344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041147442326980944, + "loss": 2.774, + "theoretical_loss": 3.8323494798360134, + "tokens_seen": 611450880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041146439317953857, + "loss": 2.7721, + "theoretical_loss": 3.832306763886927, + "tokens_seen": 611516416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114543630892678, + "loss": 2.8789, + "theoretical_loss": 3.8322640537970916, + "tokens_seen": 611581952 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411444332998997, + "loss": 2.794, + "theoretical_loss": 3.8322213495650743, + "tokens_seen": 611647488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041143430290872617, + "loss": 2.6454, + "theoretical_loss": 3.8321786511894453, + "tokens_seen": 611713024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041142427281845535, + "loss": 2.9449, + "theoretical_loss": 3.8321359586687738, + "tokens_seen": 611778560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114142427281846, + "loss": 2.8482, + "theoretical_loss": 3.83209327200163, + "tokens_seen": 611844096 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041140421263791377, + "loss": 2.7537, + "theoretical_loss": 3.832050591186584, + "tokens_seen": 611909632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041139418254764295, + "loss": 2.6385, + "theoretical_loss": 3.832007916222208, + "tokens_seen": 611975168 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041138415245737213, + "loss": 2.9391, + "theoretical_loss": 3.8319652471070724, + "tokens_seen": 612040704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113741223671013, + "loss": 2.7014, + "theoretical_loss": 3.8319225838397495, + "tokens_seen": 612106240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041136409227683055, + "loss": 2.8633, + "theoretical_loss": 3.8318799264188126, + "tokens_seen": 612171776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041135406218655967, + "loss": 2.7242, + "theoretical_loss": 3.831837274842834, + "tokens_seen": 612237312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113440320962889, + "loss": 2.6845, + "theoretical_loss": 3.8317946291103873, + "tokens_seen": 612302848 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041133400200601803, + "loss": 2.723, + "theoretical_loss": 3.8317519892200473, + "tokens_seen": 612368384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041132397191574727, + "loss": 2.8871, + "theoretical_loss": 3.8317093551703882, + "tokens_seen": 612433920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041131394182547645, + "loss": 2.9039, + "theoretical_loss": 3.8316667269599844, + "tokens_seen": 612499456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041130391173520563, + "loss": 2.7024, + "theoretical_loss": 3.831624104587413, + "tokens_seen": 612564992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112938816449348, + "loss": 2.6651, + "theoretical_loss": 3.831581488051248, + "tokens_seen": 612630528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041128385155466405, + "loss": 2.7755, + "theoretical_loss": 3.8315388773500683, + "tokens_seen": 612696064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 348068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.487290859222412, + "objective/train/theoretical_loss": 3.8314962724824495, + "objective/train/tokens_used": 633221600, + "theoretical_loss": 3.8314962724824495, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112738214643932, + "loss": 2.6193, + "theoretical_loss": 3.8314962724824495, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112637913741224, + "loss": 2.9404, + "theoretical_loss": 3.831453673446969, + "tokens_seen": 612827136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125376128385154, + "loss": 2.6889, + "theoretical_loss": 3.831411080242206, + "tokens_seen": 612892672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112437311935808, + "loss": 2.8746, + "theoretical_loss": 3.831368492866738, + "tokens_seen": 612958208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041123370110330995, + "loss": 2.7783, + "theoretical_loss": 3.831325911319145, + "tokens_seen": 613023744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041122367101303914, + "loss": 2.529, + "theoretical_loss": 3.8312833355980063, + "tokens_seen": 613089280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112136409227683, + "loss": 2.8101, + "theoretical_loss": 3.831240765701901, + "tokens_seen": 613154816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112036108324975, + "loss": 2.5207, + "theoretical_loss": 3.8311982016294115, + "tokens_seen": 613220352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111935807422267, + "loss": 2.7324, + "theoretical_loss": 3.831155643379117, + "tokens_seen": 613285888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111835506519559, + "loss": 2.7735, + "theoretical_loss": 3.831113090949601, + "tokens_seen": 613351424 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041117352056168504, + "loss": 2.91, + "theoretical_loss": 3.8310705443394433, + "tokens_seen": 613416960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111634904714143, + "loss": 3.0031, + "theoretical_loss": 3.8310280035472286, + "tokens_seen": 613482496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111534603811434, + "loss": 2.77, + "theoretical_loss": 3.830985468571538, + "tokens_seen": 613548032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041114343029087264, + "loss": 2.6051, + "theoretical_loss": 3.8309429394109573, + "tokens_seen": 613613568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111334002006018, + "loss": 2.6809, + "theoretical_loss": 3.8309004160640683, + "tokens_seen": 613679104 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411123370110331, + "loss": 2.5764, + "theoretical_loss": 3.830857898529457, + "tokens_seen": 613744640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111133400200602, + "loss": 2.8588, + "theoretical_loss": 3.8308153868057078, + "tokens_seen": 613810176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111033099297894, + "loss": 2.6019, + "theoretical_loss": 3.8307728808914066, + "tokens_seen": 613875712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041109327983951854, + "loss": 2.6243, + "theoretical_loss": 3.8307303807851394, + "tokens_seen": 613941248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110832497492478, + "loss": 2.9228, + "theoretical_loss": 3.8306878864854923, + "tokens_seen": 614006784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110732196589769, + "loss": 2.6788, + "theoretical_loss": 3.8306453979910526, + "tokens_seen": 614072320 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041106318956870614, + "loss": 2.7896, + "theoretical_loss": 3.830602915300408, + "tokens_seen": 614137856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110531594784353, + "loss": 2.8886, + "theoretical_loss": 3.8305604384121468, + "tokens_seen": 614203392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110431293881645, + "loss": 2.6726, + "theoretical_loss": 3.830517967324856, + "tokens_seen": 614268928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110330992978937, + "loss": 2.7219, + "theoretical_loss": 3.8304755020371264, + "tokens_seen": 614334464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 348575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.63563871383667, + "objective/train/theoretical_loss": 3.830433042547546, + "objective/train/tokens_used": 634860000, + "theoretical_loss": 3.830433042547546, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041102306920762287, + "loss": 2.8635, + "theoretical_loss": 3.830433042547546, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041101303911735205, + "loss": 2.7305, + "theoretical_loss": 3.8303905888547067, + "tokens_seen": 614465536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110030090270813, + "loss": 2.823, + "theoretical_loss": 3.830348140957197, + "tokens_seen": 614531072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109929789368104, + "loss": 2.6875, + "theoretical_loss": 3.830305698853609, + "tokens_seen": 614596608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041098294884653965, + "loss": 2.8067, + "theoretical_loss": 3.830263262542534, + "tokens_seen": 614662144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041097291875626877, + "loss": 2.755, + "theoretical_loss": 3.8302208320225635, + "tokens_seen": 614727680 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410962888665998, + "loss": 2.7622, + "theoretical_loss": 3.8301784072922898, + "tokens_seen": 614793216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109528585757272, + "loss": 2.7521, + "theoretical_loss": 3.8301359883503068, + "tokens_seen": 614858752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041094282848545637, + "loss": 2.606, + "theoretical_loss": 3.8300935751952077, + "tokens_seen": 614924288 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041093279839518555, + "loss": 2.7874, + "theoretical_loss": 3.8300511678255855, + "tokens_seen": 614989824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109227683049148, + "loss": 2.6015, + "theoretical_loss": 3.830008766240036, + "tokens_seen": 615055360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109127382146439, + "loss": 2.6941, + "theoretical_loss": 3.8299663704371527, + "tokens_seen": 615120896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041090270812437315, + "loss": 2.5843, + "theoretical_loss": 3.829923980415532, + "tokens_seen": 615186432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108926780341023, + "loss": 2.8617, + "theoretical_loss": 3.8298815961737693, + "tokens_seen": 615251968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108826479438315, + "loss": 2.7504, + "theoretical_loss": 3.829839217710462, + "tokens_seen": 615317504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108726178535607, + "loss": 2.6976, + "theoretical_loss": 3.8297968450242053, + "tokens_seen": 615383040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041086258776328987, + "loss": 2.874, + "theoretical_loss": 3.829754478113597, + "tokens_seen": 615448576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041085255767301905, + "loss": 2.6128, + "theoretical_loss": 3.829712116977236, + "tokens_seen": 615514112 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041084252758274823, + "loss": 2.6613, + "theoretical_loss": 3.82966976161372, + "tokens_seen": 615579648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108324974924774, + "loss": 2.7108, + "theoretical_loss": 3.8296274120216474, + "tokens_seen": 615645184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041082246740220665, + "loss": 2.5394, + "theoretical_loss": 3.829585068199618, + "tokens_seen": 615710720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108124373119358, + "loss": 2.7524, + "theoretical_loss": 3.8295427301462324, + "tokens_seen": 615776256 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410802407221665, + "loss": 2.8075, + "theoretical_loss": 3.8295003978600892, + "tokens_seen": 615841792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107923771313942, + "loss": 2.7175, + "theoretical_loss": 3.82945807133979, + "tokens_seen": 615907328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107823470411234, + "loss": 2.7832, + "theoretical_loss": 3.8294157505839364, + "tokens_seen": 615972864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 349861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8283627033233643, + "objective/train/theoretical_loss": 3.8293734355911297, + "objective/train/tokens_used": 636498400, + "theoretical_loss": 3.8293734355911297, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041077231695085256, + "loss": 2.578, + "theoretical_loss": 3.8293734355911297, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041076228686058174, + "loss": 2.726, + "theoretical_loss": 3.829331126359972, + "tokens_seen": 616103936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107522567703109, + "loss": 2.8416, + "theoretical_loss": 3.8292888228890662, + "tokens_seen": 616169472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041074222668004015, + "loss": 2.7834, + "theoretical_loss": 3.8292465251770165, + "tokens_seen": 616235008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107321965897693, + "loss": 2.5378, + "theoretical_loss": 3.829204233222425, + "tokens_seen": 616300544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107221664994985, + "loss": 2.7511, + "theoretical_loss": 3.829161947023897, + "tokens_seen": 616366080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041071213640922764, + "loss": 2.5905, + "theoretical_loss": 3.829119666580037, + "tokens_seen": 616431616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107021063189569, + "loss": 2.8605, + "theoretical_loss": 3.82907739188945, + "tokens_seen": 616497152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041069207622868606, + "loss": 2.9454, + "theoretical_loss": 3.8290351229507418, + "tokens_seen": 616562688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041068204613841524, + "loss": 2.8061, + "theoretical_loss": 3.8289928597625185, + "tokens_seen": 616628224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106720160481444, + "loss": 2.7851, + "theoretical_loss": 3.828950602323386, + "tokens_seen": 616693760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106619859578736, + "loss": 2.7758, + "theoretical_loss": 3.828908350631953, + "tokens_seen": 616759296 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041065195586760284, + "loss": 2.8322, + "theoretical_loss": 3.828866104686825, + "tokens_seen": 616824832 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410641925777332, + "loss": 2.7427, + "theoretical_loss": 3.828823864486612, + "tokens_seen": 616890368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106318956870612, + "loss": 2.854, + "theoretical_loss": 3.828781630029922, + "tokens_seen": 616955904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106218655967904, + "loss": 2.7491, + "theoretical_loss": 3.8287394013153637, + "tokens_seen": 617021440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106118355065196, + "loss": 2.5018, + "theoretical_loss": 3.828697178341547, + "tokens_seen": 617086976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041060180541624874, + "loss": 2.7982, + "theoretical_loss": 3.8286549611070813, + "tokens_seen": 617152512 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410591775325978, + "loss": 2.7268, + "theoretical_loss": 3.8286127496105777, + "tokens_seen": 617218048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105817452357071, + "loss": 2.8512, + "theoretical_loss": 3.828570543850647, + "tokens_seen": 617283584 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041057171514543634, + "loss": 2.7221, + "theoretical_loss": 3.8285283438259015, + "tokens_seen": 617349120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105616850551655, + "loss": 2.7407, + "theoretical_loss": 3.8284861495349514, + "tokens_seen": 617414656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105516549648947, + "loss": 2.6172, + "theoretical_loss": 3.828443960976411, + "tokens_seen": 617480192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105416248746239, + "loss": 2.8896, + "theoretical_loss": 3.828401778148892, + "tokens_seen": 617545728 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041053159478435307, + "loss": 2.5109, + "theoretical_loss": 3.828359601051008, + "tokens_seen": 617611264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 350619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7856407165527344, + "objective/train/theoretical_loss": 3.8283174296813733, + "objective/train/tokens_used": 638136800, + "theoretical_loss": 3.8283174296813733, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041052156469408225, + "loss": 2.6453, + "theoretical_loss": 3.8283174296813733, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105115346038115, + "loss": 2.6044, + "theoretical_loss": 3.828275264038602, + "tokens_seen": 617742336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105015045135406, + "loss": 2.5576, + "theoretical_loss": 3.828233104121309, + "tokens_seen": 617807872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041049147442326985, + "loss": 2.6203, + "theoretical_loss": 3.8281909499281097, + "tokens_seen": 617873408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041048144433299897, + "loss": 2.6868, + "theoretical_loss": 3.82814880145762, + "tokens_seen": 617938944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104714142427282, + "loss": 2.7666, + "theoretical_loss": 3.828106658708456, + "tokens_seen": 618004480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104613841524574, + "loss": 2.8267, + "theoretical_loss": 3.8280645216792344, + "tokens_seen": 618070016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041045135406218657, + "loss": 2.9681, + "theoretical_loss": 3.828022390368572, + "tokens_seen": 618135552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041044132397191575, + "loss": 2.6423, + "theoretical_loss": 3.8279802647750882, + "tokens_seen": 618201088 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410431293881645, + "loss": 2.4776, + "theoretical_loss": 3.8279381448973995, + "tokens_seen": 618266624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104212637913741, + "loss": 2.5911, + "theoretical_loss": 3.827896030734126, + "tokens_seen": 618332160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041041123370110335, + "loss": 2.4597, + "theoretical_loss": 3.8278539222838854, + "tokens_seen": 618397696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104012036108325, + "loss": 2.8388, + "theoretical_loss": 3.827811819545299, + "tokens_seen": 618463232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103911735205617, + "loss": 2.7457, + "theoretical_loss": 3.827769722516985, + "tokens_seen": 618528768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103811434302909, + "loss": 2.8415, + "theoretical_loss": 3.8277276311975656, + "tokens_seen": 618594304 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041037111334002007, + "loss": 2.8539, + "theoretical_loss": 3.8276855455856618, + "tokens_seen": 618659840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041036108324974925, + "loss": 2.7644, + "theoretical_loss": 3.8276434656798948, + "tokens_seen": 618725376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041035105315947844, + "loss": 2.6033, + "theoretical_loss": 3.8276013914788862, + "tokens_seen": 618790912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103410230692076, + "loss": 2.9456, + "theoretical_loss": 3.827559322981259, + "tokens_seen": 618856448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041033099297893685, + "loss": 2.5836, + "theoretical_loss": 3.827517260185637, + "tokens_seen": 618921984 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410320962888666, + "loss": 2.8173, + "theoretical_loss": 3.827475203090642, + "tokens_seen": 618987520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103109327983952, + "loss": 2.898, + "theoretical_loss": 3.827433151694899, + "tokens_seen": 619053056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103009027081244, + "loss": 2.7016, + "theoretical_loss": 3.827391105997033, + "tokens_seen": 619118592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102908726178536, + "loss": 2.7453, + "theoretical_loss": 3.8273490659956675, + "tokens_seen": 619184128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041028084252758276, + "loss": 2.6333, + "theoretical_loss": 3.827307031689429, + "tokens_seen": 619249664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 351768, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.970876693725586, + "objective/train/theoretical_loss": 3.8272650030769433, + "objective/train/tokens_used": 639775200, + "theoretical_loss": 3.8272650030769433, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041027081243731194, + "loss": 2.797, + "theoretical_loss": 3.8272650030769433, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102607823470411, + "loss": 2.6878, + "theoretical_loss": 3.8272229801568365, + "tokens_seen": 619380736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041025075225677035, + "loss": 2.6138, + "theoretical_loss": 3.827180962927735, + "tokens_seen": 619446272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102407221664995, + "loss": 2.7821, + "theoretical_loss": 3.827138951388267, + "tokens_seen": 619511808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102306920762287, + "loss": 2.8132, + "theoretical_loss": 3.8270969455370594, + "tokens_seen": 619577344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041022066198595784, + "loss": 2.7769, + "theoretical_loss": 3.8270549453727414, + "tokens_seen": 619642880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102106318956871, + "loss": 2.6755, + "theoretical_loss": 3.8270129508939412, + "tokens_seen": 619708416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041020060180541626, + "loss": 2.593, + "theoretical_loss": 3.8269709620992884, + "tokens_seen": 619773952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041019057171514544, + "loss": 2.873, + "theoretical_loss": 3.826928978987412, + "tokens_seen": 619839488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101805416248746, + "loss": 2.8805, + "theoretical_loss": 3.8268870015569427, + "tokens_seen": 619905024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101705115346038, + "loss": 2.6737, + "theoretical_loss": 3.826845029806511, + "tokens_seen": 619970560 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410160481444333, + "loss": 2.5944, + "theoretical_loss": 3.826803063734748, + "tokens_seen": 620036096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101504513540622, + "loss": 2.7064, + "theoretical_loss": 3.826761103340286, + "tokens_seen": 620101632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041014042126379135, + "loss": 2.834, + "theoretical_loss": 3.8267191486217556, + "tokens_seen": 620167168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101303911735206, + "loss": 2.5143, + "theoretical_loss": 3.8266771995777904, + "tokens_seen": 620232704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041012036108324976, + "loss": 2.864, + "theoretical_loss": 3.826635256207023, + "tokens_seen": 620298240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041011033099297894, + "loss": 2.5869, + "theoretical_loss": 3.8265933185080874, + "tokens_seen": 620363776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101003009027081, + "loss": 2.7903, + "theoretical_loss": 3.8265513864796166, + "tokens_seen": 620429312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100902708124373, + "loss": 2.7977, + "theoretical_loss": 3.8265094601202465, + "tokens_seen": 620494848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100802407221665, + "loss": 2.7521, + "theoretical_loss": 3.8264675394286103, + "tokens_seen": 620560384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100702106318957, + "loss": 3.0977, + "theoretical_loss": 3.8264256244033445, + "tokens_seen": 620625920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041006018054162485, + "loss": 2.8243, + "theoretical_loss": 3.826383715043085, + "tokens_seen": 620691456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100501504513541, + "loss": 2.8642, + "theoretical_loss": 3.826341811346468, + "tokens_seen": 620756992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100401203610832, + "loss": 2.6281, + "theoretical_loss": 3.8262999133121296, + "tokens_seen": 620822528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041003009027081245, + "loss": 2.935, + "theoretical_loss": 3.826258020938708, + "tokens_seen": 620888064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 352593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.924837827682495, + "objective/train/theoretical_loss": 3.82621613422484, + "objective/train/tokens_used": 641413600, + "theoretical_loss": 3.82621613422484, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041002006018054163, + "loss": 2.7235, + "theoretical_loss": 3.82621613422484, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100100300902708, + "loss": 2.9794, + "theoretical_loss": 3.8261742531691647, + "tokens_seen": 621019136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041, + "loss": 2.6264, + "theoretical_loss": 3.8261323777703207, + "tokens_seen": 621084672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040998996990972917, + "loss": 2.7152, + "theoretical_loss": 3.8260905080269465, + "tokens_seen": 621150208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040997993981945835, + "loss": 2.9183, + "theoretical_loss": 3.8260486439376824, + "tokens_seen": 621215744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099699097291876, + "loss": 2.8273, + "theoretical_loss": 3.826006785501168, + "tokens_seen": 621281280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099598796389167, + "loss": 2.8176, + "theoretical_loss": 3.8259649327160448, + "tokens_seen": 621346816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040994984954864595, + "loss": 2.6598, + "theoretical_loss": 3.8259230855809525, + "tokens_seen": 621412352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040993981945837513, + "loss": 2.8564, + "theoretical_loss": 3.825881244094533, + "tokens_seen": 621477888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099297893681043, + "loss": 2.8814, + "theoretical_loss": 3.825839408255429, + "tokens_seen": 621543424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099197592778335, + "loss": 2.97, + "theoretical_loss": 3.825797578062282, + "tokens_seen": 621608960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099097291875627, + "loss": 2.5886, + "theoretical_loss": 3.825755753513736, + "tokens_seen": 621674496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098996990972919, + "loss": 2.8166, + "theoretical_loss": 3.8257139346084332, + "tokens_seen": 621740032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098896690070211, + "loss": 2.6562, + "theoretical_loss": 3.8256721213450184, + "tokens_seen": 621805568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098796389167503, + "loss": 2.6982, + "theoretical_loss": 3.8256303137221357, + "tokens_seen": 621871104 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040986960882647945, + "loss": 2.6747, + "theoretical_loss": 3.825588511738429, + "tokens_seen": 621936640 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040985957873620864, + "loss": 2.7269, + "theoretical_loss": 3.8255467153925453, + "tokens_seen": 622002176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098495486459378, + "loss": 2.8566, + "theoretical_loss": 3.8255049246831287, + "tokens_seen": 622067712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040983951855566705, + "loss": 2.934, + "theoretical_loss": 3.825463139608827, + "tokens_seen": 622133248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098294884653962, + "loss": 2.9967, + "theoretical_loss": 3.825421360168285, + "tokens_seen": 622198784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098194583751254, + "loss": 2.5096, + "theoretical_loss": 3.8253795863601505, + "tokens_seen": 622264320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098094282848546, + "loss": 2.7312, + "theoretical_loss": 3.825337818183072, + "tokens_seen": 622329856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097993981945838, + "loss": 2.5197, + "theoretical_loss": 3.8252960556356967, + "tokens_seen": 622395392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040978936810431296, + "loss": 2.8355, + "theoretical_loss": 3.8252542987166738, + "tokens_seen": 622460928 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040977933801404214, + "loss": 2.6231, + "theoretical_loss": 3.8252125474246514, + "tokens_seen": 622526464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 353199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.111541748046875, + "objective/train/theoretical_loss": 3.8251708017582793, + "objective/train/tokens_used": 643052000, + "theoretical_loss": 3.8251708017582793, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097693079237713, + "loss": 2.7705, + "theoretical_loss": 3.8251708017582793, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040975927783350055, + "loss": 2.8652, + "theoretical_loss": 3.825129061716208, + "tokens_seen": 622657536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097492477432297, + "loss": 2.6621, + "theoretical_loss": 3.825087327297087, + "tokens_seen": 622723072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097392176529589, + "loss": 2.7638, + "theoretical_loss": 3.8250455984995684, + "tokens_seen": 622788608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040972918756268804, + "loss": 2.8562, + "theoretical_loss": 3.825003875322302, + "tokens_seen": 622854144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097191574724173, + "loss": 2.762, + "theoretical_loss": 3.8249621577639408, + "tokens_seen": 622919680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040970912738214646, + "loss": 2.8821, + "theoretical_loss": 3.8249204458231363, + "tokens_seen": 622985216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040969909729187564, + "loss": 2.8805, + "theoretical_loss": 3.8248787394985415, + "tokens_seen": 623050752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096890672016048, + "loss": 2.7553, + "theoretical_loss": 3.82483703878881, + "tokens_seen": 623116288 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409679037111334, + "loss": 2.7981, + "theoretical_loss": 3.824795343692595, + "tokens_seen": 623181824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096690070210632, + "loss": 2.5148, + "theoretical_loss": 3.8247536542085507, + "tokens_seen": 623247360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096589769307924, + "loss": 2.7058, + "theoretical_loss": 3.824711970335332, + "tokens_seen": 623312896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040964894684052155, + "loss": 2.8072, + "theoretical_loss": 3.824670292071593, + "tokens_seen": 623378432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096389167502508, + "loss": 2.7979, + "theoretical_loss": 3.82462861941599, + "tokens_seen": 623443968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040962888665997996, + "loss": 2.6498, + "theoretical_loss": 3.8245869523671794, + "tokens_seen": 623509504 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040961885656970914, + "loss": 2.844, + "theoretical_loss": 3.8245452909238167, + "tokens_seen": 623575040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096088264794383, + "loss": 2.9147, + "theoretical_loss": 3.824503635084559, + "tokens_seen": 623640576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095987963891675, + "loss": 2.7143, + "theoretical_loss": 3.8244619848480648, + "tokens_seen": 623706112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095887662988967, + "loss": 2.8396, + "theoretical_loss": 3.82442034021299, + "tokens_seen": 623771648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095787362086259, + "loss": 2.7964, + "theoretical_loss": 3.824378701177994, + "tokens_seen": 623837184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040956870611835505, + "loss": 2.6739, + "theoretical_loss": 3.8243370677417357, + "tokens_seen": 623902720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095586760280843, + "loss": 2.6736, + "theoretical_loss": 3.8242954399028743, + "tokens_seen": 623968256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095486459378134, + "loss": 2.6681, + "theoretical_loss": 3.824253817660069, + "tokens_seen": 624033792 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040953861584754265, + "loss": 2.8982, + "theoretical_loss": 3.82421220101198, + "tokens_seen": 624099328 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040952858575727183, + "loss": 2.914, + "theoretical_loss": 3.8241705899572684, + "tokens_seen": 624164864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 354273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4973878860473633, + "objective/train/theoretical_loss": 3.8241289844945947, + "objective/train/tokens_used": 644690400, + "theoretical_loss": 3.8241289844945947, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409518555667001, + "loss": 2.747, + "theoretical_loss": 3.8241289844945947, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095085255767302, + "loss": 2.6438, + "theoretical_loss": 3.8240873846226204, + "tokens_seen": 624295936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040949849548645937, + "loss": 2.6574, + "theoretical_loss": 3.8240457903400085, + "tokens_seen": 624361472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040948846539618855, + "loss": 2.902, + "theoretical_loss": 3.82400420164542, + "tokens_seen": 624427008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094784353059178, + "loss": 2.9509, + "theoretical_loss": 3.823962618537519, + "tokens_seen": 624492544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094684052156469, + "loss": 2.9318, + "theoretical_loss": 3.8239210410149678, + "tokens_seen": 624558080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040945837512537615, + "loss": 2.6767, + "theoretical_loss": 3.823879469076431, + "tokens_seen": 624623616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040944834503510533, + "loss": 2.8474, + "theoretical_loss": 3.8238379027205727, + "tokens_seen": 624689152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094383149448345, + "loss": 2.7381, + "theoretical_loss": 3.823796341946058, + "tokens_seen": 624754688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094282848545637, + "loss": 2.8622, + "theoretical_loss": 3.823754786751551, + "tokens_seen": 624820224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094182547642929, + "loss": 2.8514, + "theoretical_loss": 3.8237132371357188, + "tokens_seen": 624885760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040940822467402206, + "loss": 2.8813, + "theoretical_loss": 3.823671693097226, + "tokens_seen": 624951296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093981945837513, + "loss": 2.614, + "theoretical_loss": 3.8236301546347407, + "tokens_seen": 625016832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093881644934804, + "loss": 2.7085, + "theoretical_loss": 3.8235886217469286, + "tokens_seen": 625082368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040937813440320965, + "loss": 2.7141, + "theoretical_loss": 3.8235470944324588, + "tokens_seen": 625147904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093681043129388, + "loss": 2.7833, + "theoretical_loss": 3.823505572689997, + "tokens_seen": 625213440 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409358074222668, + "loss": 2.8488, + "theoretical_loss": 3.8234640565182136, + "tokens_seen": 625278976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093480441323972, + "loss": 2.8908, + "theoretical_loss": 3.8234225459157773, + "tokens_seen": 625344512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093380140421264, + "loss": 2.879, + "theoretical_loss": 3.823381040881356, + "tokens_seen": 625410048 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040932798395185556, + "loss": 2.656, + "theoretical_loss": 3.8233395414136204, + "tokens_seen": 625475584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093179538615848, + "loss": 2.5753, + "theoretical_loss": 3.8232980475112415, + "tokens_seen": 625541120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093079237713139, + "loss": 2.5492, + "theoretical_loss": 3.8232565591728886, + "tokens_seen": 625606656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040929789368104316, + "loss": 2.7042, + "theoretical_loss": 3.8232150763972337, + "tokens_seen": 625672192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092878635907723, + "loss": 2.7082, + "theoretical_loss": 3.8231735991829483, + "tokens_seen": 625737728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092778335005015, + "loss": 2.8102, + "theoretical_loss": 3.8231321275287042, + "tokens_seen": 625803264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 354855, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.032952308654785, + "objective/train/theoretical_loss": 3.8230906614331737, + "objective/train/tokens_used": 646328800, + "theoretical_loss": 3.8230906614331737, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092678034102307, + "loss": 2.8881, + "theoretical_loss": 3.8230906614331737, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092577733199599, + "loss": 2.8983, + "theoretical_loss": 3.823049200895031, + "tokens_seen": 625934336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040924774322968906, + "loss": 2.857, + "theoretical_loss": 3.823007745912948, + "tokens_seen": 625999872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040923771313941824, + "loss": 3.0152, + "theoretical_loss": 3.8229662964855997, + "tokens_seen": 626065408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092276830491474, + "loss": 2.9206, + "theoretical_loss": 3.82292485261166, + "tokens_seen": 626130944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040921765295887666, + "loss": 2.6842, + "theoretical_loss": 3.822883414289804, + "tokens_seen": 626196480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092076228686058, + "loss": 2.9224, + "theoretical_loss": 3.8228419815187062, + "tokens_seen": 626262016 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409197592778335, + "loss": 2.6684, + "theoretical_loss": 3.822800554297043, + "tokens_seen": 626327552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040918756268806415, + "loss": 2.9722, + "theoretical_loss": 3.8227591326234904, + "tokens_seen": 626393088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091775325977934, + "loss": 2.6933, + "theoretical_loss": 3.822717716496725, + "tokens_seen": 626458624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040916750250752257, + "loss": 2.9309, + "theoretical_loss": 3.822676305915424, + "tokens_seen": 626524160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040915747241725175, + "loss": 2.9623, + "theoretical_loss": 3.822634900878265, + "tokens_seen": 626589696 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409147442326981, + "loss": 2.7984, + "theoretical_loss": 3.8225935013839254, + "tokens_seen": 626655232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040913741223671016, + "loss": 2.7617, + "theoretical_loss": 3.8225521074310844, + "tokens_seen": 626720768 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040912738214643934, + "loss": 2.9621, + "theoretical_loss": 3.82251071901842, + "tokens_seen": 626786304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091173520561685, + "loss": 2.7483, + "theoretical_loss": 3.8224693361446134, + "tokens_seen": 626851840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091073219658977, + "loss": 2.4055, + "theoretical_loss": 3.822427958808342, + "tokens_seen": 626917376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090972918756269, + "loss": 2.9295, + "theoretical_loss": 3.822386587008287, + "tokens_seen": 626982912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090872617853561, + "loss": 2.787, + "theoretical_loss": 3.82234522074313, + "tokens_seen": 627048448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040907723169508525, + "loss": 2.6379, + "theoretical_loss": 3.8223038600115506, + "tokens_seen": 627113984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090672016048145, + "loss": 2.8665, + "theoretical_loss": 3.8222625048122314, + "tokens_seen": 627179520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090571715145436, + "loss": 2.844, + "theoretical_loss": 3.8222211551438545, + "tokens_seen": 627245056 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040904714142427285, + "loss": 2.5677, + "theoretical_loss": 3.8221798110051024, + "tokens_seen": 627310592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040903711133400203, + "loss": 2.8402, + "theoretical_loss": 3.8221384723946574, + "tokens_seen": 627376128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090270812437312, + "loss": 2.7906, + "theoretical_loss": 3.8220971393112033, + "tokens_seen": 627441664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 356151, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0492939949035645, + "objective/train/theoretical_loss": 3.8220558117534242, + "objective/train/tokens_used": 647967200, + "theoretical_loss": 3.8220558117534242, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090170511534604, + "loss": 2.7642, + "theoretical_loss": 3.8220558117534242, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040900702106318957, + "loss": 2.7001, + "theoretical_loss": 3.8220144897200035, + "tokens_seen": 627572736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040899699097291875, + "loss": 3.154, + "theoretical_loss": 3.821973173209628, + "tokens_seen": 627638272 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408986960882648, + "loss": 2.9608, + "theoretical_loss": 3.82193186222098, + "tokens_seen": 627703808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089769307923771, + "loss": 2.7454, + "theoretical_loss": 3.821890556752748, + "tokens_seen": 627769344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040896690070210635, + "loss": 2.8077, + "theoretical_loss": 3.821849256803617, + "tokens_seen": 627834880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040895687061183553, + "loss": 2.6292, + "theoretical_loss": 3.8218079623722723, + "tokens_seen": 627900416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089468405215647, + "loss": 2.6008, + "theoretical_loss": 3.821766673457403, + "tokens_seen": 627965952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089368104312939, + "loss": 2.8147, + "theoretical_loss": 3.8217253900576953, + "tokens_seen": 628031488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089267803410231, + "loss": 2.9192, + "theoretical_loss": 3.821684112171837, + "tokens_seen": 628097024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040891675025075226, + "loss": 3.0209, + "theoretical_loss": 3.8216428397985176, + "tokens_seen": 628162560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089067201604815, + "loss": 2.6749, + "theoretical_loss": 3.8216015729364248, + "tokens_seen": 628228096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088966900702106, + "loss": 2.847, + "theoretical_loss": 3.821560311584248, + "tokens_seen": 628293632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040888665997993985, + "loss": 2.6515, + "theoretical_loss": 3.8215190557406773, + "tokens_seen": 628359168 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408876629889669, + "loss": 2.6926, + "theoretical_loss": 3.821477805404403, + "tokens_seen": 628424704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088665997993982, + "loss": 2.9088, + "theoretical_loss": 3.8214365605741154, + "tokens_seen": 628490240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088565697091274, + "loss": 2.7691, + "theoretical_loss": 3.8213953212485055, + "tokens_seen": 628555776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088465396188566, + "loss": 2.8476, + "theoretical_loss": 3.821354087426265, + "tokens_seen": 628621312 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040883650952858576, + "loss": 2.6684, + "theoretical_loss": 3.821312859106085, + "tokens_seen": 628686848 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408826479438315, + "loss": 2.7181, + "theoretical_loss": 3.8212716362866597, + "tokens_seen": 628752384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088164493480441, + "loss": 2.5384, + "theoretical_loss": 3.8212304189666804, + "tokens_seen": 628817920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040880641925777336, + "loss": 2.6511, + "theoretical_loss": 3.8211892071448403, + "tokens_seen": 628883456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087963891675025, + "loss": 2.672, + "theoretical_loss": 3.8211480008198344, + "tokens_seen": 628948992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087863590772317, + "loss": 2.6401, + "theoretical_loss": 3.821106799990356, + "tokens_seen": 629014528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087763289869609, + "loss": 2.6822, + "theoretical_loss": 3.8210656046550997, + "tokens_seen": 629080064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 356825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.88836932182312, + "objective/train/theoretical_loss": 3.821024414812761, + "objective/train/tokens_used": 649605600, + "theoretical_loss": 3.821024414812761, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087662988966901, + "loss": 2.7571, + "theoretical_loss": 3.821024414812761, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040875626880641926, + "loss": 2.8849, + "theoretical_loss": 3.8209832304620353, + "tokens_seen": 629211136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040874623871614844, + "loss": 2.8463, + "theoretical_loss": 3.8209420516016177, + "tokens_seen": 629276672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087362086258776, + "loss": 2.5645, + "theoretical_loss": 3.820900878230206, + "tokens_seen": 629342208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040872617853560686, + "loss": 2.9021, + "theoretical_loss": 3.8208597103464963, + "tokens_seen": 629407744 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408716148445336, + "loss": 2.8184, + "theoretical_loss": 3.820818547949186, + "tokens_seen": 629473280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087061183550652, + "loss": 2.9188, + "theoretical_loss": 3.820777391036973, + "tokens_seen": 629538816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040869608826479435, + "loss": 3.0061, + "theoretical_loss": 3.8207362396085554, + "tokens_seen": 629604352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086860581745236, + "loss": 2.9378, + "theoretical_loss": 3.820695093662631, + "tokens_seen": 629669888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040867602808425277, + "loss": 2.6738, + "theoretical_loss": 3.8206539531979002, + "tokens_seen": 629735424 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040866599799398195, + "loss": 2.6967, + "theoretical_loss": 3.8206128182130623, + "tokens_seen": 629800960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040865596790371113, + "loss": 2.6859, + "theoretical_loss": 3.8205716887068166, + "tokens_seen": 629866496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040864593781344036, + "loss": 2.8031, + "theoretical_loss": 3.8205305646778633, + "tokens_seen": 629932032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086359077231695, + "loss": 2.8133, + "theoretical_loss": 3.820489446124905, + "tokens_seen": 629997568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086258776328987, + "loss": 2.8721, + "theoretical_loss": 3.8204483330466408, + "tokens_seen": 630063104 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040861584754262785, + "loss": 3.052, + "theoretical_loss": 3.820407225441774, + "tokens_seen": 630128640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086058174523571, + "loss": 3.0175, + "theoretical_loss": 3.820366123309006, + "tokens_seen": 630194176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040859578736208627, + "loss": 2.6538, + "theoretical_loss": 3.82032502664704, + "tokens_seen": 630259712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040858575727181545, + "loss": 2.8017, + "theoretical_loss": 3.8202839354545786, + "tokens_seen": 630325248 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040857572718154463, + "loss": 2.8723, + "theoretical_loss": 3.8202428497303247, + "tokens_seen": 630390784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085656970912738, + "loss": 2.8212, + "theoretical_loss": 3.820201769472984, + "tokens_seen": 630456320 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408555667001003, + "loss": 2.9297, + "theoretical_loss": 3.8201606946812596, + "tokens_seen": 630521856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040854563691073223, + "loss": 2.9594, + "theoretical_loss": 3.820119625353856, + "tokens_seen": 630587392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040853560682046136, + "loss": 2.7654, + "theoretical_loss": 3.8200785614894794, + "tokens_seen": 630652928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085255767301906, + "loss": 2.7078, + "theoretical_loss": 3.820037503086836, + "tokens_seen": 630718464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 357395, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.229945182800293, + "objective/train/theoretical_loss": 3.81999645014463, + "objective/train/tokens_used": 651244000, + "theoretical_loss": 3.81999645014463, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085155466399197, + "loss": 2.9155, + "theoretical_loss": 3.81999645014463, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040850551654964895, + "loss": 2.7578, + "theoretical_loss": 3.81995540266157, + "tokens_seen": 630849536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040849548645937814, + "loss": 2.7972, + "theoretical_loss": 3.819914360636362, + "tokens_seen": 630915072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084854563691073, + "loss": 2.9271, + "theoretical_loss": 3.8198733240677134, + "tokens_seen": 630980608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084754262788365, + "loss": 2.8088, + "theoretical_loss": 3.8198322929543327, + "tokens_seen": 631046144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040846539618856573, + "loss": 2.8873, + "theoretical_loss": 3.819791267294928, + "tokens_seen": 631111680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040845536609829486, + "loss": 2.764, + "theoretical_loss": 3.8197502470882077, + "tokens_seen": 631177216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084453360080241, + "loss": 2.6695, + "theoretical_loss": 3.8197092323328814, + "tokens_seen": 631242752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084353059177532, + "loss": 3.0271, + "theoretical_loss": 3.8196682230276595, + "tokens_seen": 631308288 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040842527582748246, + "loss": 2.8736, + "theoretical_loss": 3.8196272191712506, + "tokens_seen": 631373824 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040841524573721164, + "loss": 2.8502, + "theoretical_loss": 3.8195862207623668, + "tokens_seen": 631439360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084052156469408, + "loss": 2.6863, + "theoretical_loss": 3.8195452277997184, + "tokens_seen": 631504896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040839518555667005, + "loss": 2.6358, + "theoretical_loss": 3.819504240282016, + "tokens_seen": 631570432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083851554663992, + "loss": 2.8913, + "theoretical_loss": 3.819463258207973, + "tokens_seen": 631635968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083751253761284, + "loss": 2.8457, + "theoretical_loss": 3.8194222815763004, + "tokens_seen": 631701504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083650952858576, + "loss": 3.0551, + "theoretical_loss": 3.8193813103857126, + "tokens_seen": 631767040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083550651955868, + "loss": 2.7654, + "theoretical_loss": 3.8193403446349206, + "tokens_seen": 631832576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040834503510531596, + "loss": 2.8818, + "theoretical_loss": 3.8192993843226395, + "tokens_seen": 631898112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083350050150452, + "loss": 2.9639, + "theoretical_loss": 3.8192584294475833, + "tokens_seen": 631963648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083249749247743, + "loss": 2.812, + "theoretical_loss": 3.8192174800084664, + "tokens_seen": 632029184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040831494483450356, + "loss": 2.7267, + "theoretical_loss": 3.819176536004003, + "tokens_seen": 632094720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083049147442327, + "loss": 2.8316, + "theoretical_loss": 3.8191355974329095, + "tokens_seen": 632160256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082948846539619, + "loss": 2.7239, + "theoretical_loss": 3.8190946642939014, + "tokens_seen": 632225792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082848545636911, + "loss": 2.6075, + "theoretical_loss": 3.819053736585694, + "tokens_seen": 632291328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082748244734203, + "loss": 2.7152, + "theoretical_loss": 3.8190128143070057, + "tokens_seen": 632356864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 358555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.62202525138855, + "objective/train/theoretical_loss": 3.818971897456552, + "objective/train/tokens_used": 652882400, + "theoretical_loss": 3.818971897456552, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040826479438314946, + "loss": 2.9621, + "theoretical_loss": 3.818971897456552, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040825476429287864, + "loss": 2.7414, + "theoretical_loss": 3.818930986033052, + "tokens_seen": 632487936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082447342026078, + "loss": 2.7112, + "theoretical_loss": 3.8188900800352226, + "tokens_seen": 632553472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040823470411233706, + "loss": 2.9688, + "theoretical_loss": 3.8188491794617825, + "tokens_seen": 632619008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082246740220662, + "loss": 2.9359, + "theoretical_loss": 3.81880828431145, + "tokens_seen": 632684544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082146439317954, + "loss": 2.7537, + "theoretical_loss": 3.8187673945829457, + "tokens_seen": 632750080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040820461384152455, + "loss": 2.583, + "theoretical_loss": 3.8187265102749883, + "tokens_seen": 632815616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081945837512538, + "loss": 2.8925, + "theoretical_loss": 3.8186856313862982, + "tokens_seen": 632881152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040818455366098297, + "loss": 2.487, + "theoretical_loss": 3.818644757915597, + "tokens_seen": 632946688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040817452357071215, + "loss": 2.7202, + "theoretical_loss": 3.8186038898616035, + "tokens_seen": 633012224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040816449348044133, + "loss": 2.8124, + "theoretical_loss": 3.8185630272230413, + "tokens_seen": 633077760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040815446339017056, + "loss": 2.66, + "theoretical_loss": 3.8185221699986314, + "tokens_seen": 633143296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081444332998997, + "loss": 2.8485, + "theoretical_loss": 3.818481318187096, + "tokens_seen": 633208832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081344032096289, + "loss": 2.8472, + "theoretical_loss": 3.8184404717871585, + "tokens_seen": 633274368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040812437311935805, + "loss": 2.766, + "theoretical_loss": 3.8183996307975416, + "tokens_seen": 633339904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081143430290873, + "loss": 2.5663, + "theoretical_loss": 3.8183587952169686, + "tokens_seen": 633405440 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040810431293881647, + "loss": 2.7984, + "theoretical_loss": 3.818317965044165, + "tokens_seen": 633470976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040809428284854565, + "loss": 2.9312, + "theoretical_loss": 3.818277140277854, + "tokens_seen": 633536512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040808425275827483, + "loss": 2.6212, + "theoretical_loss": 3.8182363209167605, + "tokens_seen": 633602048 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408074222668004, + "loss": 2.8187, + "theoretical_loss": 3.8181955069596105, + "tokens_seen": 633667584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080641925777332, + "loss": 2.7484, + "theoretical_loss": 3.81815469840513, + "tokens_seen": 633733120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040805416248746243, + "loss": 2.8055, + "theoretical_loss": 3.8181138952520444, + "tokens_seen": 633798656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804413239719156, + "loss": 2.805, + "theoretical_loss": 3.8180730974990813, + "tokens_seen": 633864192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080341023069208, + "loss": 2.8455, + "theoretical_loss": 3.8180323051449667, + "tokens_seen": 633929728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080240722166499, + "loss": 2.824, + "theoretical_loss": 3.817991518188429, + "tokens_seen": 633995264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 359090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7171945571899414, + "objective/train/theoretical_loss": 3.8179507366281964, + "objective/train/tokens_used": 654520800, + "theoretical_loss": 3.8179507366281964, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040801404212637915, + "loss": 2.8011, + "theoretical_loss": 3.8179507366281964, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040800401203610834, + "loss": 2.6352, + "theoretical_loss": 3.817909960462996, + "tokens_seen": 634126336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079939819458375, + "loss": 2.7008, + "theoretical_loss": 3.817869189691558, + "tokens_seen": 634191872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079839518555667, + "loss": 2.4876, + "theoretical_loss": 3.8178284243126113, + "tokens_seen": 634257408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040797392176529593, + "loss": 2.5169, + "theoretical_loss": 3.8177876643248845, + "tokens_seen": 634322944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040796389167502506, + "loss": 2.5934, + "theoretical_loss": 3.8177469097271093, + "tokens_seen": 634388480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079538615847543, + "loss": 2.5588, + "theoretical_loss": 3.817706160518015, + "tokens_seen": 634454016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079438314944834, + "loss": 2.6794, + "theoretical_loss": 3.8176654166963337, + "tokens_seen": 634519552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040793380140421266, + "loss": 2.5933, + "theoretical_loss": 3.8176246782607963, + "tokens_seen": 634585088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040792377131394184, + "loss": 2.6201, + "theoretical_loss": 3.817583945210134, + "tokens_seen": 634650624 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407913741223671, + "loss": 2.9393, + "theoretical_loss": 3.8175432175430797, + "tokens_seen": 634716160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079037111334002, + "loss": 2.8609, + "theoretical_loss": 3.817502495258366, + "tokens_seen": 634781696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078936810431294, + "loss": 2.7891, + "theoretical_loss": 3.817461778354727, + "tokens_seen": 634847232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040788365095285856, + "loss": 2.9239, + "theoretical_loss": 3.8174210668308945, + "tokens_seen": 634912768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078736208625878, + "loss": 2.7264, + "theoretical_loss": 3.8173803606856036, + "tokens_seen": 634978304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078635907723169, + "loss": 2.7941, + "theoretical_loss": 3.8173396599175886, + "tokens_seen": 635043840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040785356068204616, + "loss": 2.8307, + "theoretical_loss": 3.817298964525584, + "tokens_seen": 635109376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078435305917753, + "loss": 2.8384, + "theoretical_loss": 3.817258274508325, + "tokens_seen": 635174912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078335005015045, + "loss": 2.7362, + "theoretical_loss": 3.8172175898645477, + "tokens_seen": 635240448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078234704112337, + "loss": 2.5734, + "theoretical_loss": 3.817176910592988, + "tokens_seen": 635305984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078134403209629, + "loss": 2.732, + "theoretical_loss": 3.8171362366923827, + "tokens_seen": 635371520 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040780341023069207, + "loss": 2.6684, + "theoretical_loss": 3.817095568161468, + "tokens_seen": 635437056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077933801404213, + "loss": 2.8199, + "theoretical_loss": 3.817054904998983, + "tokens_seen": 635502592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040778335005015043, + "loss": 2.601, + "theoretical_loss": 3.817014247203664, + "tokens_seen": 635568128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040777331995987966, + "loss": 2.6932, + "theoretical_loss": 3.8169735947742494, + "tokens_seen": 635633664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 360366, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.813488483428955, + "objective/train/theoretical_loss": 3.8169329477094784, + "objective/train/tokens_used": 656159200, + "theoretical_loss": 3.8169329477094784, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077632898696088, + "loss": 2.7224, + "theoretical_loss": 3.8169329477094784, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407753259779338, + "loss": 2.7715, + "theoretical_loss": 3.81689230600809, + "tokens_seen": 635764736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077432296890672, + "loss": 2.6614, + "theoretical_loss": 3.8168516696688233, + "tokens_seen": 635830272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077331995987964, + "loss": 2.7141, + "theoretical_loss": 3.8168110386904193, + "tokens_seen": 635895808 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040772316950852557, + "loss": 2.8513, + "theoretical_loss": 3.8167704130716174, + "tokens_seen": 635961344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040771313941825475, + "loss": 2.8096, + "theoretical_loss": 3.8167297928111585, + "tokens_seen": 636026880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040770310932798393, + "loss": 2.5866, + "theoretical_loss": 3.8166891779077847, + "tokens_seen": 636092416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040769307923771317, + "loss": 3.1336, + "theoretical_loss": 3.816648568360236, + "tokens_seen": 636157952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076830491474423, + "loss": 2.6937, + "theoretical_loss": 3.8166079641672566, + "tokens_seen": 636223488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040767301905717153, + "loss": 2.7533, + "theoretical_loss": 3.8165673653275873, + "tokens_seen": 636289024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076629889669007, + "loss": 2.8758, + "theoretical_loss": 3.816526771839972, + "tokens_seen": 636354560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076529588766299, + "loss": 2.6713, + "theoretical_loss": 3.816486183703154, + "tokens_seen": 636420096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076429287863591, + "loss": 3.0028, + "theoretical_loss": 3.8164456009158765, + "tokens_seen": 636485632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040763289869608825, + "loss": 2.6476, + "theoretical_loss": 3.816405023476884, + "tokens_seen": 636551168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076228686058175, + "loss": 3.0361, + "theoretical_loss": 3.8163644513849215, + "tokens_seen": 636616704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040761283851554667, + "loss": 2.4477, + "theoretical_loss": 3.816323884638734, + "tokens_seen": 636682240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040760280842527585, + "loss": 2.7836, + "theoretical_loss": 3.8162833232370668, + "tokens_seen": 636747776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040759277833500503, + "loss": 2.7696, + "theoretical_loss": 3.8162427671786654, + "tokens_seen": 636813312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075827482447342, + "loss": 2.6237, + "theoretical_loss": 3.8162022164622766, + "tokens_seen": 636878848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075727181544634, + "loss": 2.7036, + "theoretical_loss": 3.8161616710866473, + "tokens_seen": 636944384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040756268806419263, + "loss": 2.7311, + "theoretical_loss": 3.8161211310505245, + "tokens_seen": 637009920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040755265797392176, + "loss": 2.8929, + "theoretical_loss": 3.8160805963526556, + "tokens_seen": 637075456 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407542627883651, + "loss": 2.9721, + "theoretical_loss": 3.816040066991789, + "tokens_seen": 637140992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075325977933801, + "loss": 2.9736, + "theoretical_loss": 3.815999542966673, + "tokens_seen": 637206528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040752256770310935, + "loss": 2.5887, + "theoretical_loss": 3.8159590242760566, + "tokens_seen": 637272064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 360972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.695178270339966, + "objective/train/theoretical_loss": 3.8159185109186886, + "objective/train/tokens_used": 657797600, + "theoretical_loss": 3.8159185109186886, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040751253761283854, + "loss": 2.7188, + "theoretical_loss": 3.8159185109186886, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075025075225677, + "loss": 2.8754, + "theoretical_loss": 3.815878002893319, + "tokens_seen": 637403136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074924774322969, + "loss": 2.8012, + "theoretical_loss": 3.8158375001986986, + "tokens_seen": 637468672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040748244734202613, + "loss": 2.7555, + "theoretical_loss": 3.815797002833577, + "tokens_seen": 637534208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040747241725175526, + "loss": 2.6793, + "theoretical_loss": 3.815756510796705, + "tokens_seen": 637599744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074623871614845, + "loss": 2.8207, + "theoretical_loss": 3.8157160240868353, + "tokens_seen": 637665280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074523570712136, + "loss": 2.9315, + "theoretical_loss": 3.815675542702719, + "tokens_seen": 637730816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040744232698094286, + "loss": 2.6283, + "theoretical_loss": 3.8156350666431074, + "tokens_seen": 637796352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040743229689067204, + "loss": 2.8794, + "theoretical_loss": 3.815594595906755, + "tokens_seen": 637861888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074222668004012, + "loss": 2.7645, + "theoretical_loss": 3.815554130492414, + "tokens_seen": 637927424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074122367101304, + "loss": 2.61, + "theoretical_loss": 3.815513670398837, + "tokens_seen": 637992960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074022066198596, + "loss": 2.7538, + "theoretical_loss": 3.8154732156247793, + "tokens_seen": 638058496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040739217652958876, + "loss": 2.7017, + "theoretical_loss": 3.8154327661689944, + "tokens_seen": 638124032 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407382146439318, + "loss": 2.7127, + "theoretical_loss": 3.815392322030238, + "tokens_seen": 638189568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073721163490471, + "loss": 2.6691, + "theoretical_loss": 3.8153518832072644, + "tokens_seen": 638255104 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040736208625877636, + "loss": 2.5609, + "theoretical_loss": 3.8153114496988296, + "tokens_seen": 638320640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073520561685055, + "loss": 2.9655, + "theoretical_loss": 3.8152710215036896, + "tokens_seen": 638386176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073420260782347, + "loss": 2.825, + "theoretical_loss": 3.8152305986206, + "tokens_seen": 638451712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073319959879639, + "loss": 2.7397, + "theoretical_loss": 3.815190181048319, + "tokens_seen": 638517248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073219658976931, + "loss": 2.8857, + "theoretical_loss": 3.815149768785603, + "tokens_seen": 638582784 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040731193580742227, + "loss": 2.8068, + "theoretical_loss": 3.81510936183121, + "tokens_seen": 638648320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073019057171515, + "loss": 2.6231, + "theoretical_loss": 3.8150689601838983, + "tokens_seen": 638713856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040729187562688063, + "loss": 2.9175, + "theoretical_loss": 3.815028563842426, + "tokens_seen": 638779392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040728184553660986, + "loss": 3.0054, + "theoretical_loss": 3.8149881728055526, + "tokens_seen": 638844928 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407271815446339, + "loss": 2.8376, + "theoretical_loss": 3.814947787072037, + "tokens_seen": 638910464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 361973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6586904525756836, + "objective/train/theoretical_loss": 3.814907406640639, + "objective/train/tokens_used": 659436000, + "theoretical_loss": 3.814907406640639, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072617853560682, + "loss": 2.7195, + "theoretical_loss": 3.814907406640639, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072517552657974, + "loss": 2.8213, + "theoretical_loss": 3.8148670315101185, + "tokens_seen": 639041536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072417251755266, + "loss": 2.882, + "theoretical_loss": 3.8148266616792377, + "tokens_seen": 639107072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040723169508525577, + "loss": 2.8047, + "theoretical_loss": 3.814786297146756, + "tokens_seen": 639172608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040722166499498495, + "loss": 2.7604, + "theoretical_loss": 3.8147459379114346, + "tokens_seen": 639238144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040721163490471413, + "loss": 2.8299, + "theoretical_loss": 3.8147055839720365, + "tokens_seen": 639303680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040720160481444337, + "loss": 2.827, + "theoretical_loss": 3.8146652353273236, + "tokens_seen": 639369216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071915747241725, + "loss": 3.0141, + "theoretical_loss": 3.8146248919760586, + "tokens_seen": 639434752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040718154463390173, + "loss": 2.6655, + "theoretical_loss": 3.8145845539170047, + "tokens_seen": 639500288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071715145436309, + "loss": 2.9756, + "theoretical_loss": 3.8145442211489256, + "tokens_seen": 639565824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071614844533601, + "loss": 2.6459, + "theoretical_loss": 3.8145038936705844, + "tokens_seen": 639631360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040715145436308927, + "loss": 2.7277, + "theoretical_loss": 3.814463571480746, + "tokens_seen": 639696896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040714142427281845, + "loss": 2.6967, + "theoretical_loss": 3.814423254578176, + "tokens_seen": 639762432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040713139418254763, + "loss": 2.9031, + "theoretical_loss": 3.8143829429616383, + "tokens_seen": 639827968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040712136409227687, + "loss": 2.8567, + "theoretical_loss": 3.814342636629899, + "tokens_seen": 639893504 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407111334002006, + "loss": 2.6368, + "theoretical_loss": 3.814302335581724, + "tokens_seen": 639959040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040710130391173523, + "loss": 2.7911, + "theoretical_loss": 3.81426203981588, + "tokens_seen": 640024576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040709127382146436, + "loss": 2.9786, + "theoretical_loss": 3.814221749331134, + "tokens_seen": 640090112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070812437311936, + "loss": 2.7101, + "theoretical_loss": 3.8141814641262526, + "tokens_seen": 640155648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070712136409228, + "loss": 2.7521, + "theoretical_loss": 3.814141184200004, + "tokens_seen": 640221184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040706118355065196, + "loss": 2.8041, + "theoretical_loss": 3.814100909551156, + "tokens_seen": 640286720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040705115346038114, + "loss": 2.8164, + "theoretical_loss": 3.814060640178478, + "tokens_seen": 640352256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070411233701103, + "loss": 2.9017, + "theoretical_loss": 3.8140203760807374, + "tokens_seen": 640417792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070310932798395, + "loss": 2.7351, + "theoretical_loss": 3.8139801172567047, + "tokens_seen": 640483328 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040702106318956874, + "loss": 2.7543, + "theoretical_loss": 3.8139398637051496, + "tokens_seen": 640548864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 362406, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6734325885772705, + "objective/train/theoretical_loss": 3.813899615424841, + "objective/train/tokens_used": 661074400, + "theoretical_loss": 3.813899615424841, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040701103309929786, + "loss": 2.9096, + "theoretical_loss": 3.813899615424841, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070010030090271, + "loss": 2.8807, + "theoretical_loss": 3.8138593724145515, + "tokens_seen": 640679936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069909729187563, + "loss": 2.6374, + "theoretical_loss": 3.81381913467305, + "tokens_seen": 640745472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040698094282848546, + "loss": 2.6867, + "theoretical_loss": 3.8137789021991093, + "tokens_seen": 640811008 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040697091273821464, + "loss": 2.7622, + "theoretical_loss": 3.813738674991501, + "tokens_seen": 640876544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069608826479438, + "loss": 2.7671, + "theoretical_loss": 3.813698453048997, + "tokens_seen": 640942080 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406950852557673, + "loss": 2.7282, + "theoretical_loss": 3.81365823637037, + "tokens_seen": 641007616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040694082246740224, + "loss": 2.8956, + "theoretical_loss": 3.813618024954393, + "tokens_seen": 641073152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040693079237713137, + "loss": 2.8595, + "theoretical_loss": 3.81357781879984, + "tokens_seen": 641138688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069207622868606, + "loss": 2.797, + "theoretical_loss": 3.8135376179054843, + "tokens_seen": 641204224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040691073219658973, + "loss": 2.9285, + "theoretical_loss": 3.8134974222700997, + "tokens_seen": 641269760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040690070210631896, + "loss": 2.9507, + "theoretical_loss": 3.813457231892462, + "tokens_seen": 641335296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068906720160482, + "loss": 2.9627, + "theoretical_loss": 3.8134170467713453, + "tokens_seen": 641400832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068806419257773, + "loss": 2.7184, + "theoretical_loss": 3.813376866905526, + "tokens_seen": 641466368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040687061183550656, + "loss": 2.9675, + "theoretical_loss": 3.81333669229378, + "tokens_seen": 641531904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068605817452357, + "loss": 2.7346, + "theoretical_loss": 3.813296522934883, + "tokens_seen": 641597440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068505516549649, + "loss": 2.7329, + "theoretical_loss": 3.813256358827611, + "tokens_seen": 641662976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068405215646941, + "loss": 2.6317, + "theoretical_loss": 3.8132161999707437, + "tokens_seen": 641728512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068304914744233, + "loss": 2.8005, + "theoretical_loss": 3.813176046363056, + "tokens_seen": 641794048 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040682046138415247, + "loss": 2.8122, + "theoretical_loss": 3.8131358980033276, + "tokens_seen": 641859584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068104312938817, + "loss": 2.8557, + "theoretical_loss": 3.8130957548903357, + "tokens_seen": 641925120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040680040120361083, + "loss": 2.8691, + "theoretical_loss": 3.81305561702286, + "tokens_seen": 641990656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040679037111334006, + "loss": 2.6403, + "theoretical_loss": 3.8130154843996795, + "tokens_seen": 642056192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067803410230692, + "loss": 2.8516, + "theoretical_loss": 3.812975357019573, + "tokens_seen": 642121728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067703109327984, + "loss": 3.0307, + "theoretical_loss": 3.812935234881322, + "tokens_seen": 642187264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 363472, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8334643840789795, + "objective/train/theoretical_loss": 3.8128951179837056, + "objective/train/tokens_used": 662712800, + "theoretical_loss": 3.8128951179837056, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067602808425276, + "loss": 2.7628, + "theoretical_loss": 3.8128951179837056, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067502507522568, + "loss": 2.6521, + "theoretical_loss": 3.812855006325505, + "tokens_seen": 642318336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040674022066198597, + "loss": 2.7719, + "theoretical_loss": 3.8128148999055016, + "tokens_seen": 642383872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040673019057171515, + "loss": 2.9862, + "theoretical_loss": 3.8127747987224767, + "tokens_seen": 642449408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040672016048144433, + "loss": 2.7238, + "theoretical_loss": 3.812734702775213, + "tokens_seen": 642514944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040671013039117357, + "loss": 2.7985, + "theoretical_loss": 3.8126946120624927, + "tokens_seen": 642580480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067001003009027, + "loss": 2.9992, + "theoretical_loss": 3.8126545265830982, + "tokens_seen": 642646016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040669007021063193, + "loss": 2.7378, + "theoretical_loss": 3.8126144463358136, + "tokens_seen": 642711552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066800401203611, + "loss": 2.7071, + "theoretical_loss": 3.812574371319422, + "tokens_seen": 642777088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066700100300903, + "loss": 2.7973, + "theoretical_loss": 3.812534301532707, + "tokens_seen": 642842624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040665997993981947, + "loss": 2.8874, + "theoretical_loss": 3.8124942369744543, + "tokens_seen": 642908160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040664994984954865, + "loss": 2.7893, + "theoretical_loss": 3.812454177643448, + "tokens_seen": 642973696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040663991975927783, + "loss": 2.7979, + "theoretical_loss": 3.812414123538474, + "tokens_seen": 643039232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040662988966900707, + "loss": 2.8078, + "theoretical_loss": 3.812374074658317, + "tokens_seen": 643104768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066198595787362, + "loss": 2.8899, + "theoretical_loss": 3.812334031001764, + "tokens_seen": 643170304 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040660982948846543, + "loss": 2.6655, + "theoretical_loss": 3.812293992567602, + "tokens_seen": 643235840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040659979939819456, + "loss": 2.7146, + "theoretical_loss": 3.8122539593546163, + "tokens_seen": 643301376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004065897693079238, + "loss": 2.9341, + "theoretical_loss": 3.8122139313615953, + "tokens_seen": 643366912 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406579739217653, + "loss": 2.856, + "theoretical_loss": 3.8121739085873267, + "tokens_seen": 643432448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040656970912738216, + "loss": 2.7503, + "theoretical_loss": 3.812133891030598, + "tokens_seen": 643497984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040655967903711134, + "loss": 2.8333, + "theoretical_loss": 3.812093878690199, + "tokens_seen": 643563520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065496489468405, + "loss": 2.6505, + "theoretical_loss": 3.8120538715649177, + "tokens_seen": 643629056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065396188565697, + "loss": 2.6991, + "theoretical_loss": 3.8120138696535437, + "tokens_seen": 643694592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040652958876629894, + "loss": 2.8435, + "theoretical_loss": 3.8119738729548667, + "tokens_seen": 643760128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040651955867602806, + "loss": 2.9121, + "theoretical_loss": 3.811933881467677, + "tokens_seen": 643825664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 364407, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.769031047821045, + "objective/train/theoretical_loss": 3.8118938951907646, + "objective/train/tokens_used": 664351200, + "theoretical_loss": 3.8118938951907646, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065095285857573, + "loss": 2.9188, + "theoretical_loss": 3.8118938951907646, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064994984954865, + "loss": 3.0245, + "theoretical_loss": 3.8118539141229215, + "tokens_seen": 643956736 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040648946840521566, + "loss": 2.8493, + "theoretical_loss": 3.811813938262939, + "tokens_seen": 644022272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040647943831494484, + "loss": 2.7306, + "theoretical_loss": 3.811773967609608, + "tokens_seen": 644087808 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406469408224674, + "loss": 2.5553, + "theoretical_loss": 3.811734002161721, + "tokens_seen": 644153344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064593781344032, + "loss": 2.7385, + "theoretical_loss": 3.8116940419180705, + "tokens_seen": 644218880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040644934804413244, + "loss": 2.7768, + "theoretical_loss": 3.81165408687745, + "tokens_seen": 644284416 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040643931795386157, + "loss": 2.9223, + "theoretical_loss": 3.8116141370386525, + "tokens_seen": 644349952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064292878635908, + "loss": 2.5762, + "theoretical_loss": 3.811574192400472, + "tokens_seen": 644415488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040641925777331993, + "loss": 2.6792, + "theoretical_loss": 3.8115342529617022, + "tokens_seen": 644481024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040640922768304916, + "loss": 2.7865, + "theoretical_loss": 3.811494318721139, + "tokens_seen": 644546560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040639919759277834, + "loss": 2.9013, + "theoretical_loss": 3.811454389677576, + "tokens_seen": 644612096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063891675025075, + "loss": 2.5841, + "theoretical_loss": 3.811414465829809, + "tokens_seen": 644677632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063791374122367, + "loss": 2.7939, + "theoretical_loss": 3.8113745471766336, + "tokens_seen": 644743168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063691073219659, + "loss": 3.0081, + "theoretical_loss": 3.811334633716847, + "tokens_seen": 644808704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040635907723169507, + "loss": 2.5528, + "theoretical_loss": 3.811294725449245, + "tokens_seen": 644874240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063490471414243, + "loss": 2.7421, + "theoretical_loss": 3.811254822372624, + "tokens_seen": 644939776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040633901705115343, + "loss": 2.7564, + "theoretical_loss": 3.811214924485783, + "tokens_seen": 645005312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040632898696088267, + "loss": 2.8748, + "theoretical_loss": 3.811175031787519, + "tokens_seen": 645070848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040631895687061185, + "loss": 2.7806, + "theoretical_loss": 3.81113514427663, + "tokens_seen": 645136384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040630892678034103, + "loss": 2.9969, + "theoretical_loss": 3.8110952619519143, + "tokens_seen": 645201920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062988966900702, + "loss": 2.7653, + "theoretical_loss": 3.811055384812172, + "tokens_seen": 645267456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062888665997994, + "loss": 2.8918, + "theoretical_loss": 3.8110155128562018, + "tokens_seen": 645332992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040627883650952857, + "loss": 2.7826, + "theoretical_loss": 3.8109756460828037, + "tokens_seen": 645398528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062688064192578, + "loss": 2.9599, + "theoretical_loss": 3.810935784490778, + "tokens_seen": 645464064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 365032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9448306560516357, + "objective/train/theoretical_loss": 3.8108959280789243, + "objective/train/tokens_used": 665989600, + "theoretical_loss": 3.8108959280789243, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040625877632898693, + "loss": 2.8312, + "theoretical_loss": 3.8108959280789243, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040624874623871617, + "loss": 2.8855, + "theoretical_loss": 3.8108560768460453, + "tokens_seen": 645595136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062387161484453, + "loss": 2.8467, + "theoretical_loss": 3.810816230790942, + "tokens_seen": 645660672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040622868605817453, + "loss": 2.6674, + "theoretical_loss": 3.8107763899124145, + "tokens_seen": 645726208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062186559679037, + "loss": 2.8258, + "theoretical_loss": 3.8107365542092673, + "tokens_seen": 645791744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062086258776329, + "loss": 2.9253, + "theoretical_loss": 3.8106967236803015, + "tokens_seen": 645857280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061985957873621, + "loss": 2.7386, + "theoretical_loss": 3.8106568983243205, + "tokens_seen": 645922816 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040618856569709126, + "loss": 2.7275, + "theoretical_loss": 3.8106170781401287, + "tokens_seen": 645988352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040617853560682044, + "loss": 2.8876, + "theoretical_loss": 3.8105772631265284, + "tokens_seen": 646053888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040616850551654967, + "loss": 2.9051, + "theoretical_loss": 3.8105374532823246, + "tokens_seen": 646119424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061584754262788, + "loss": 2.9001, + "theoretical_loss": 3.8104976486063213, + "tokens_seen": 646184960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040614844533600804, + "loss": 2.9, + "theoretical_loss": 3.8104578490973244, + "tokens_seen": 646250496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040613841524573727, + "loss": 2.8382, + "theoretical_loss": 3.810418054754139, + "tokens_seen": 646316032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061283851554664, + "loss": 2.8047, + "theoretical_loss": 3.81037826557557, + "tokens_seen": 646381568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040611835506519563, + "loss": 2.7919, + "theoretical_loss": 3.810338481560425, + "tokens_seen": 646447104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040610832497492476, + "loss": 2.9111, + "theoretical_loss": 3.8102987027075095, + "tokens_seen": 646512640 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406098294884654, + "loss": 2.8586, + "theoretical_loss": 3.810258929015631, + "tokens_seen": 646578176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060882647943832, + "loss": 2.678, + "theoretical_loss": 3.8102191604835967, + "tokens_seen": 646643712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040607823470411236, + "loss": 2.781, + "theoretical_loss": 3.8101793971102147, + "tokens_seen": 646709248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040606820461384154, + "loss": 2.914, + "theoretical_loss": 3.8101396388942925, + "tokens_seen": 646774784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060581745235707, + "loss": 2.7807, + "theoretical_loss": 3.810099885834639, + "tokens_seen": 646840320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060481444332999, + "loss": 2.8564, + "theoretical_loss": 3.810060137930064, + "tokens_seen": 646905856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040603811434302914, + "loss": 2.7308, + "theoretical_loss": 3.810020395179375, + "tokens_seen": 646971392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040602808425275826, + "loss": 2.898, + "theoretical_loss": 3.8099806575813835, + "tokens_seen": 647036928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060180541624875, + "loss": 2.8808, + "theoretical_loss": 3.809940925134899, + "tokens_seen": 647102464 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 365530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.693211317062378, + "objective/train/theoretical_loss": 3.809901197838732, + "objective/train/tokens_used": 667628000, + "theoretical_loss": 3.809901197838732, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060080240722167, + "loss": 2.7837, + "theoretical_loss": 3.809901197838732, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040599799398194586, + "loss": 3.0024, + "theoretical_loss": 3.8098614756916933, + "tokens_seen": 647233536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040598796389167504, + "loss": 2.8533, + "theoretical_loss": 3.8098217586925944, + "tokens_seen": 647299072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059779338014042, + "loss": 2.7603, + "theoretical_loss": 3.8097820468402475, + "tokens_seen": 647364608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059679037111334, + "loss": 2.6107, + "theoretical_loss": 3.8097423401334636, + "tokens_seen": 647430144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040595787362086264, + "loss": 2.7351, + "theoretical_loss": 3.809702638571056, + "tokens_seen": 647495680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040594784353059177, + "loss": 2.6653, + "theoretical_loss": 3.8096629421518378, + "tokens_seen": 647561216 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405937813440321, + "loss": 3.0185, + "theoretical_loss": 3.809623250874622, + "tokens_seen": 647626752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040592778335005013, + "loss": 2.7992, + "theoretical_loss": 3.8095835647382223, + "tokens_seen": 647692288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040591775325977936, + "loss": 2.6663, + "theoretical_loss": 3.809543883741453, + "tokens_seen": 647757824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040590772316950854, + "loss": 2.6354, + "theoretical_loss": 3.809504207883128, + "tokens_seen": 647823360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058976930792377, + "loss": 2.7836, + "theoretical_loss": 3.8094645371620626, + "tokens_seen": 647888896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058876629889669, + "loss": 2.7201, + "theoretical_loss": 3.809424871577072, + "tokens_seen": 647954432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058776328986961, + "loss": 2.7792, + "theoretical_loss": 3.8093852111269726, + "tokens_seen": 648019968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040586760280842527, + "loss": 2.7333, + "theoretical_loss": 3.809345555810579, + "tokens_seen": 648085504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058575727181545, + "loss": 2.7829, + "theoretical_loss": 3.8093059056267085, + "tokens_seen": 648151040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040584754262788363, + "loss": 2.8198, + "theoretical_loss": 3.8092662605741783, + "tokens_seen": 648216576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040583751253761287, + "loss": 2.7442, + "theoretical_loss": 3.8092266206518044, + "tokens_seen": 648282112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040582748244734205, + "loss": 2.7689, + "theoretical_loss": 3.809186985858406, + "tokens_seen": 648347648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040581745235707123, + "loss": 2.9267, + "theoretical_loss": 3.8091473561928, + "tokens_seen": 648413184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058074222668004, + "loss": 2.9537, + "theoretical_loss": 3.8091077316538056, + "tokens_seen": 648478720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057973921765296, + "loss": 2.9389, + "theoretical_loss": 3.809068112240241, + "tokens_seen": 648544256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040578736208625877, + "loss": 2.7254, + "theoretical_loss": 3.8090284979509255, + "tokens_seen": 648609792 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405777331995988, + "loss": 2.8984, + "theoretical_loss": 3.8089888887846786, + "tokens_seen": 648675328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040576730190571713, + "loss": 2.8705, + "theoretical_loss": 3.808949284740321, + "tokens_seen": 648740864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 366782, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.094092845916748, + "objective/train/theoretical_loss": 3.8089096858166718, + "objective/train/tokens_used": 669266400, + "theoretical_loss": 3.8089096858166718, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040575727181544637, + "loss": 2.7799, + "theoretical_loss": 3.8089096858166718, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057472417251755, + "loss": 2.9153, + "theoretical_loss": 3.808870092012553, + "tokens_seen": 648871936 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040573721163490473, + "loss": 2.8236, + "theoretical_loss": 3.808830503326785, + "tokens_seen": 648937472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057271815446339, + "loss": 2.5253, + "theoretical_loss": 3.8087909197581897, + "tokens_seen": 649003008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057171514543631, + "loss": 3.0232, + "theoretical_loss": 3.8087513413055887, + "tokens_seen": 649068544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057071213640923, + "loss": 2.9203, + "theoretical_loss": 3.8087117679678046, + "tokens_seen": 649134080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040569709127382146, + "loss": 2.7979, + "theoretical_loss": 3.8086721997436603, + "tokens_seen": 649199616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040568706118355064, + "loss": 2.9185, + "theoretical_loss": 3.8086326366319785, + "tokens_seen": 649265152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056770310932799, + "loss": 2.5927, + "theoretical_loss": 3.808593078631583, + "tokens_seen": 649330688 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405667001003009, + "loss": 2.7122, + "theoretical_loss": 3.808553525741297, + "tokens_seen": 649396224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040565697091273824, + "loss": 2.9182, + "theoretical_loss": 3.808513977959946, + "tokens_seen": 649461760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056469408224674, + "loss": 2.899, + "theoretical_loss": 3.8084744352863535, + "tokens_seen": 649527296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056369107321966, + "loss": 2.7249, + "theoretical_loss": 3.808434897719345, + "tokens_seen": 649592832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056268806419258, + "loss": 2.9955, + "theoretical_loss": 3.8083953652577462, + "tokens_seen": 649658368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040561685055165496, + "loss": 2.591, + "theoretical_loss": 3.8083558379003826, + "tokens_seen": 649723904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040560682046138414, + "loss": 2.8997, + "theoretical_loss": 3.8083163156460804, + "tokens_seen": 649789440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055967903711134, + "loss": 2.8928, + "theoretical_loss": 3.808276798493666, + "tokens_seen": 649854976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055867602808425, + "loss": 2.8441, + "theoretical_loss": 3.808237286441967, + "tokens_seen": 649920512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040557673019057174, + "loss": 2.9768, + "theoretical_loss": 3.8081977794898103, + "tokens_seen": 649986048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040556670010030087, + "loss": 2.5429, + "theoretical_loss": 3.808158277636024, + "tokens_seen": 650051584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055566700100301, + "loss": 2.8073, + "theoretical_loss": 3.808118780879436, + "tokens_seen": 650117120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055466399197593, + "loss": 2.8952, + "theoretical_loss": 3.8080792892188753, + "tokens_seen": 650182656 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040553660982948846, + "loss": 2.7382, + "theoretical_loss": 3.8080398026531697, + "tokens_seen": 650248192 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040552657973921764, + "loss": 2.7826, + "theoretical_loss": 3.80800032118115, + "tokens_seen": 650313728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055165496489469, + "loss": 2.7656, + "theoretical_loss": 3.807960844801644, + "tokens_seen": 650379264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 367488, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.001251459121704, + "objective/train/theoretical_loss": 3.8079213735134845, + "objective/train/tokens_used": 670904800, + "theoretical_loss": 3.8079213735134845, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405506519558676, + "loss": 2.835, + "theoretical_loss": 3.8079213735134845, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040549648946840524, + "loss": 2.6539, + "theoretical_loss": 3.8078819073154997, + "tokens_seen": 650510336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040548645937813437, + "loss": 2.7046, + "theoretical_loss": 3.807842446206521, + "tokens_seen": 650575872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054764292878636, + "loss": 2.928, + "theoretical_loss": 3.8078029901853805, + "tokens_seen": 650641408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054663991975928, + "loss": 2.6514, + "theoretical_loss": 3.8077635392509084, + "tokens_seen": 650706944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040545636910732197, + "loss": 3.1162, + "theoretical_loss": 3.807724093401938, + "tokens_seen": 650772480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040544633901705115, + "loss": 2.582, + "theoretical_loss": 3.8076846526373007, + "tokens_seen": 650838016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040543630892678033, + "loss": 2.9161, + "theoretical_loss": 3.8076452169558306, + "tokens_seen": 650903552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054262788365095, + "loss": 2.8318, + "theoretical_loss": 3.8076057863563593, + "tokens_seen": 650969088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040541624874623874, + "loss": 2.7556, + "theoretical_loss": 3.807566360837722, + "tokens_seen": 651034624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040540621865596787, + "loss": 2.7484, + "theoretical_loss": 3.807526940398751, + "tokens_seen": 651100160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053961885656971, + "loss": 2.7967, + "theoretical_loss": 3.807487525038282, + "tokens_seen": 651165696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053861584754263, + "loss": 2.9202, + "theoretical_loss": 3.8074481147551484, + "tokens_seen": 651231232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040537612838515547, + "loss": 2.7285, + "theoretical_loss": 3.807408709548187, + "tokens_seen": 651296768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053660982948847, + "loss": 2.8267, + "theoretical_loss": 3.8073693094162318, + "tokens_seen": 651362304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040535606820461383, + "loss": 2.7598, + "theoretical_loss": 3.807329914358119, + "tokens_seen": 651427840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040534603811434307, + "loss": 2.7406, + "theoretical_loss": 3.807290524372686, + "tokens_seen": 651493376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040533600802407225, + "loss": 2.5667, + "theoretical_loss": 3.807251139458767, + "tokens_seen": 651558912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040532597793380143, + "loss": 2.6472, + "theoretical_loss": 3.8072117596152015, + "tokens_seen": 651624448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053159478435306, + "loss": 2.6189, + "theoretical_loss": 3.807172384840826, + "tokens_seen": 651689984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053059177532598, + "loss": 2.5579, + "theoretical_loss": 3.807133015134478, + "tokens_seen": 651755520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040529588766298897, + "loss": 2.6739, + "theoretical_loss": 3.807093650494996, + "tokens_seen": 651821056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052858575727182, + "loss": 2.8714, + "theoretical_loss": 3.8070542909212186, + "tokens_seen": 651886592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040527582748244733, + "loss": 2.9979, + "theoretical_loss": 3.807014936411984, + "tokens_seen": 651952128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040526579739217657, + "loss": 2.6012, + "theoretical_loss": 3.806975586966133, + "tokens_seen": 652017664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 368763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9794094562530518, + "objective/train/theoretical_loss": 3.8069362425825037, + "objective/train/tokens_used": 672543200, + "theoretical_loss": 3.8069362425825037, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052557673019057, + "loss": 2.7193, + "theoretical_loss": 3.8069362425825037, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040524573721163493, + "loss": 2.9792, + "theoretical_loss": 3.806896903259937, + "tokens_seen": 652148736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052357071213641, + "loss": 2.8212, + "theoretical_loss": 3.8068575689972737, + "tokens_seen": 652214272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052256770310933, + "loss": 2.8342, + "theoretical_loss": 3.806818239793354, + "tokens_seen": 652279808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052156469408225, + "loss": 2.4792, + "theoretical_loss": 3.8067789156470195, + "tokens_seen": 652345344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040520561685055166, + "loss": 3.0418, + "theoretical_loss": 3.8067395965571116, + "tokens_seen": 652410880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040519558676028084, + "loss": 2.7229, + "theoretical_loss": 3.8067002825224723, + "tokens_seen": 652476416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051855566700101, + "loss": 2.9192, + "theoretical_loss": 3.806660973541944, + "tokens_seen": 652541952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051755265797392, + "loss": 2.7193, + "theoretical_loss": 3.80662166961437, + "tokens_seen": 652607488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040516549648946844, + "loss": 2.719, + "theoretical_loss": 3.8065823707385924, + "tokens_seen": 652673024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051554663991976, + "loss": 2.6547, + "theoretical_loss": 3.8065430769134556, + "tokens_seen": 652738560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051454363089268, + "loss": 2.7879, + "theoretical_loss": 3.8065037881378028, + "tokens_seen": 652804096 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405135406218656, + "loss": 2.9661, + "theoretical_loss": 3.8064645044104792, + "tokens_seen": 652869632 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040512537612838516, + "loss": 2.6646, + "theoretical_loss": 3.8064252257303286, + "tokens_seen": 652935168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040511534603811434, + "loss": 2.6163, + "theoretical_loss": 3.8063859520961967, + "tokens_seen": 653000704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051053159478436, + "loss": 2.9179, + "theoretical_loss": 3.8063466835069284, + "tokens_seen": 653066240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050952858575727, + "loss": 2.8113, + "theoretical_loss": 3.8063074199613696, + "tokens_seen": 653131776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040508525576730194, + "loss": 2.7895, + "theoretical_loss": 3.806268161458367, + "tokens_seen": 653197312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040507522567703107, + "loss": 2.6098, + "theoretical_loss": 3.8062289079967666, + "tokens_seen": 653262848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050651955867603, + "loss": 2.6069, + "theoretical_loss": 3.8061896595754154, + "tokens_seen": 653328384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050551654964895, + "loss": 2.7038, + "theoretical_loss": 3.8061504161931605, + "tokens_seen": 653393920 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040504513540621866, + "loss": 2.5877, + "theoretical_loss": 3.8061111778488503, + "tokens_seen": 653459456 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040503510531594784, + "loss": 2.7855, + "theoretical_loss": 3.8060719445413325, + "tokens_seen": 653524992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050250752256771, + "loss": 2.6148, + "theoretical_loss": 3.8060327162694563, + "tokens_seen": 653590528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050150451354062, + "loss": 2.8388, + "theoretical_loss": 3.8059934930320685, + "tokens_seen": 653656064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 369902, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6009209156036377, + "objective/train/theoretical_loss": 3.8059542748280206, + "objective/train/tokens_used": 674181600, + "theoretical_loss": 3.8059542748280206, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040500501504513544, + "loss": 2.5162, + "theoretical_loss": 3.8059542748280206, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040499498495486457, + "loss": 2.681, + "theoretical_loss": 3.805915061656161, + "tokens_seen": 653787136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049849548645938, + "loss": 2.7516, + "theoretical_loss": 3.8058758535153396, + "tokens_seen": 653852672 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404974924774323, + "loss": 2.6446, + "theoretical_loss": 3.805836650404407, + "tokens_seen": 653918208 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040496489468405217, + "loss": 2.659, + "theoretical_loss": 3.8057974523222144, + "tokens_seen": 653983744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040495486459378135, + "loss": 2.7514, + "theoretical_loss": 3.8057582592676127, + "tokens_seen": 654049280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040494483450351053, + "loss": 2.8106, + "theoretical_loss": 3.8057190712394524, + "tokens_seen": 654114816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049348044132397, + "loss": 2.5477, + "theoretical_loss": 3.805679888236587, + "tokens_seen": 654180352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040492477432296894, + "loss": 2.889, + "theoretical_loss": 3.805640710257867, + "tokens_seen": 654245888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040491474423269807, + "loss": 2.886, + "theoretical_loss": 3.805601537302147, + "tokens_seen": 654311424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049047141424273, + "loss": 2.6806, + "theoretical_loss": 3.8055623693682783, + "tokens_seen": 654376960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040489468405215643, + "loss": 2.8927, + "theoretical_loss": 3.8055232064551148, + "tokens_seen": 654442496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040488465396188567, + "loss": 3.0574, + "theoretical_loss": 3.8054840485615102, + "tokens_seen": 654508032 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040487462387161485, + "loss": 2.7444, + "theoretical_loss": 3.8054448956863194, + "tokens_seen": 654573568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040486459378134403, + "loss": 2.9624, + "theoretical_loss": 3.8054057478283956, + "tokens_seen": 654639104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048545636910732, + "loss": 2.9824, + "theoretical_loss": 3.805366604986595, + "tokens_seen": 654704640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040484453360080245, + "loss": 2.9506, + "theoretical_loss": 3.8053274671597714, + "tokens_seen": 654770176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048345035105316, + "loss": 2.8364, + "theoretical_loss": 3.8052883343467814, + "tokens_seen": 654835712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048244734202608, + "loss": 2.8075, + "theoretical_loss": 3.805249206546481, + "tokens_seen": 654901248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040481444332998994, + "loss": 2.7262, + "theoretical_loss": 3.8052100837577263, + "tokens_seen": 654966784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040480441323971917, + "loss": 2.7897, + "theoretical_loss": 3.805170965979374, + "tokens_seen": 655032320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040479438314944835, + "loss": 2.6729, + "theoretical_loss": 3.805131853210282, + "tokens_seen": 655097856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040478435305917753, + "loss": 2.7982, + "theoretical_loss": 3.8050927454493064, + "tokens_seen": 655163392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047743229689067, + "loss": 2.6755, + "theoretical_loss": 3.8050536426953063, + "tokens_seen": 655228928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047642928786359, + "loss": 2.6891, + "theoretical_loss": 3.8050145449471398, + "tokens_seen": 655294464 + }, + { + "debugging/Self-BLEU-5": NaN, + "debugging/distinct-1-grams": 0.0, + "debugging/distinct-2-grams": 0.0, + "debugging/entropy-1-grams": 0.0, + "debugging/entropy-2-grams": 0.0, + "debugging/length": NaN, + "debugging/num_segments": 0, + "debugging/score": NaN, + "debugging/score_std": NaN, + "epoch": 0.2, + "objective/train/docs_used": 370281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.031588315963745, + "objective/train/theoretical_loss": 3.8049754522036645, + "objective/train/tokens_used": 675820000, + "theoretical_loss": 3.8049754522036645, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047542627883651, + "loss": 2.844, + "theoretical_loss": 3.8049754522036645, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047442326980943, + "loss": 2.7745, + "theoretical_loss": 3.8049363644637406, + "tokens_seen": 655425536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040473420260782344, + "loss": 2.6569, + "theoretical_loss": 3.8048972817262268, + "tokens_seen": 655491072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047241725175527, + "loss": 3.1308, + "theoretical_loss": 3.8048582039899834, + "tokens_seen": 655556608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047141424272818, + "loss": 2.8553, + "theoretical_loss": 3.8048191312538697, + "tokens_seen": 655622144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040470411233701104, + "loss": 2.5904, + "theoretical_loss": 3.8047800635167466, + "tokens_seen": 655687680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046940822467402, + "loss": 2.6663, + "theoretical_loss": 3.804741000777476, + "tokens_seen": 655753216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046840521564694, + "loss": 3.0068, + "theoretical_loss": 3.8047019430349174, + "tokens_seen": 655818752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046740220661986, + "loss": 2.9107, + "theoretical_loss": 3.8046628902879336, + "tokens_seen": 655884288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046639919759278, + "loss": 2.843, + "theoretical_loss": 3.8046238425353858, + "tokens_seen": 655949824 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404653961885657, + "loss": 2.7753, + "theoretical_loss": 3.804584799776137, + "tokens_seen": 656015360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046439317953862, + "loss": 2.8462, + "theoretical_loss": 3.80454576200905, + "tokens_seen": 656080896 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040463390170511536, + "loss": 2.6241, + "theoretical_loss": 3.8045067292329877, + "tokens_seen": 656146432 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040462387161484454, + "loss": 2.8776, + "theoretical_loss": 3.8044677014468133, + "tokens_seen": 656211968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046138415245738, + "loss": 2.8266, + "theoretical_loss": 3.804428678649391, + "tokens_seen": 656277504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046038114343029, + "loss": 2.913, + "theoretical_loss": 3.8043896608395844, + "tokens_seen": 656343040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040459378134403214, + "loss": 2.7933, + "theoretical_loss": 3.804350648016259, + "tokens_seen": 656408576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040458375125376127, + "loss": 2.8499, + "theoretical_loss": 3.8043116401782795, + "tokens_seen": 656474112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045737211634905, + "loss": 2.7959, + "theoretical_loss": 3.8042726373245115, + "tokens_seen": 656539648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045636910732197, + "loss": 2.9439, + "theoretical_loss": 3.8042336394538196, + "tokens_seen": 656605184 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040455366098294886, + "loss": 2.9103, + "theoretical_loss": 3.8041946465650707, + "tokens_seen": 656670720 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040454363089267804, + "loss": 2.5819, + "theoretical_loss": 3.8041556586571312, + "tokens_seen": 656736256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045336008024073, + "loss": 2.8263, + "theoretical_loss": 3.804116675728868, + "tokens_seen": 656801792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045235707121364, + "loss": 2.9633, + "theoretical_loss": 3.8040776977791486, + "tokens_seen": 656867328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040451354062186564, + "loss": 2.9724, + "theoretical_loss": 3.8040387248068397, + "tokens_seen": 656932864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 371094, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9736223220825195, + "objective/train/theoretical_loss": 3.8039997568108097, + "objective/train/tokens_used": 677458400, + "theoretical_loss": 3.8039997568108097, + "tokens_seen": 656998400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040450351053159477, + "loss": 2.8304, + "theoretical_loss": 3.8039997568108097, + "tokens_seen": 656998400 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404493480441324, + "loss": 2.9945, + "theoretical_loss": 3.803960793789927, + "tokens_seen": 657063936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044834503510532, + "loss": 2.7311, + "theoretical_loss": 3.8039218357430604, + "tokens_seen": 657129472 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040447342026078237, + "loss": 2.7709, + "theoretical_loss": 3.8038828826690785, + "tokens_seen": 657195008 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040446339017051155, + "loss": 2.8639, + "theoretical_loss": 3.803843934566851, + "tokens_seen": 657260544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040445336008024073, + "loss": 2.6372, + "theoretical_loss": 3.8038049914352476, + "tokens_seen": 657326080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044433299899699, + "loss": 2.75, + "theoretical_loss": 3.803766053273139, + "tokens_seen": 657391616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040443329989969915, + "loss": 3.0652, + "theoretical_loss": 3.8037271200793947, + "tokens_seen": 657457152 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040442326980942827, + "loss": 2.8485, + "theoretical_loss": 3.803688191852886, + "tokens_seen": 657522688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044132397191575, + "loss": 2.6473, + "theoretical_loss": 3.803649268592485, + "tokens_seen": 657588224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040440320962888663, + "loss": 2.7618, + "theoretical_loss": 3.8036103502970624, + "tokens_seen": 657653760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040439317953861587, + "loss": 2.9614, + "theoretical_loss": 3.8035714369654903, + "tokens_seen": 657719296 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040438314944834505, + "loss": 2.8396, + "theoretical_loss": 3.8035325285966413, + "tokens_seen": 657784832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040437311935807423, + "loss": 2.9647, + "theoretical_loss": 3.8034936251893883, + "tokens_seen": 657850368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043630892678034, + "loss": 2.844, + "theoretical_loss": 3.8034547267426038, + "tokens_seen": 657915904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040435305917753265, + "loss": 2.7157, + "theoretical_loss": 3.803415833255162, + "tokens_seen": 657981440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043430290872618, + "loss": 2.8558, + "theoretical_loss": 3.803376944725936, + "tokens_seen": 658046976 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404332998996991, + "loss": 2.6925, + "theoretical_loss": 3.803338061153801, + "tokens_seen": 658112512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040432296890672014, + "loss": 2.8953, + "theoretical_loss": 3.8032991825376303, + "tokens_seen": 658178048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040431293881644937, + "loss": 2.8478, + "theoretical_loss": 3.8032603088763, + "tokens_seen": 658243584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040430290872617855, + "loss": 2.6926, + "theoretical_loss": 3.803221440168685, + "tokens_seen": 658309120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040429287863590773, + "loss": 2.7789, + "theoretical_loss": 3.8031825764136604, + "tokens_seen": 658374656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042828485456369, + "loss": 2.8289, + "theoretical_loss": 3.8031437176101033, + "tokens_seen": 658440192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042728184553661, + "loss": 2.8906, + "theoretical_loss": 3.80310486375689, + "tokens_seen": 658505728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042627883650953, + "loss": 2.972, + "theoretical_loss": 3.803066014852896, + "tokens_seen": 658571264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 371720, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1109814643859863, + "objective/train/theoretical_loss": 3.8030271708970003, + "objective/train/tokens_used": 679096800, + "theoretical_loss": 3.8030271708970003, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042527582748245, + "loss": 2.7763, + "theoretical_loss": 3.8030271708970003, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040424272818455364, + "loss": 2.7485, + "theoretical_loss": 3.802988331888079, + "tokens_seen": 658702336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042326980942829, + "loss": 3.001, + "theoretical_loss": 3.8029494978250105, + "tokens_seen": 658767872 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404222668004012, + "loss": 2.9512, + "theoretical_loss": 3.802910668706673, + "tokens_seen": 658833408 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040421263791374124, + "loss": 2.7941, + "theoretical_loss": 3.802871844531945, + "tokens_seen": 658898944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042026078234704, + "loss": 2.8382, + "theoretical_loss": 3.802833025299706, + "tokens_seen": 658964480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041925777331996, + "loss": 3.1284, + "theoretical_loss": 3.802794211008835, + "tokens_seen": 659030016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041825476429288, + "loss": 2.8907, + "theoretical_loss": 3.802755401658211, + "tokens_seen": 659095552 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404172517552658, + "loss": 2.5927, + "theoretical_loss": 3.802716597246715, + "tokens_seen": 659161088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040416248746238714, + "loss": 2.8932, + "theoretical_loss": 3.802677797773227, + "tokens_seen": 659226624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041524573721164, + "loss": 2.766, + "theoretical_loss": 3.802639003236629, + "tokens_seen": 659292160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041424272818455, + "loss": 2.8984, + "theoretical_loss": 3.8026002136358006, + "tokens_seen": 659357696 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040413239719157474, + "loss": 2.9041, + "theoretical_loss": 3.8025614289696232, + "tokens_seen": 659423232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041223671013039, + "loss": 2.748, + "theoretical_loss": 3.8025226492369804, + "tokens_seen": 659488768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041123370110331, + "loss": 2.8719, + "theoretical_loss": 3.802483874436753, + "tokens_seen": 659554304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041023069207623, + "loss": 2.6527, + "theoretical_loss": 3.8024451045678243, + "tokens_seen": 659619840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040409227683049147, + "loss": 2.8325, + "theoretical_loss": 3.8024063396290773, + "tokens_seen": 659685376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040408224674022065, + "loss": 2.7487, + "theoretical_loss": 3.8023675796193945, + "tokens_seen": 659750912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040722166499499, + "loss": 2.8749, + "theoretical_loss": 3.8023288245376614, + "tokens_seen": 659816448 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404062186559679, + "loss": 2.8572, + "theoretical_loss": 3.8022900743827606, + "tokens_seen": 659881984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040405215646940824, + "loss": 2.9801, + "theoretical_loss": 3.8022513291535764, + "tokens_seen": 659947520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040404212637913737, + "loss": 2.92, + "theoretical_loss": 3.8022125888489944, + "tokens_seen": 660013056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040320962888666, + "loss": 2.8742, + "theoretical_loss": 3.8021738534678997, + "tokens_seen": 660078592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040220661985958, + "loss": 2.8415, + "theoretical_loss": 3.8021351230091773, + "tokens_seen": 660144128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040401203610832497, + "loss": 3.0197, + "theoretical_loss": 3.802096397471714, + "tokens_seen": 660209664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 373016, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.935034990310669, + "objective/train/theoretical_loss": 3.802057676854395, + "objective/train/tokens_used": 680735200, + "theoretical_loss": 3.802057676854395, + "tokens_seen": 660275200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040400200601805415, + "loss": 2.6929, + "theoretical_loss": 3.802057676854395, + "tokens_seen": 660275200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039919759277834, + "loss": 2.8014, + "theoretical_loss": 3.802018961156108, + "tokens_seen": 660340736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039819458375125, + "loss": 3.053, + "theoretical_loss": 3.801980250375739, + "tokens_seen": 660406272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040397191574724175, + "loss": 2.898, + "theoretical_loss": 3.801941544512176, + "tokens_seen": 660471808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039618856569709, + "loss": 2.8489, + "theoretical_loss": 3.8019028435643065, + "tokens_seen": 660537344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039518555667001, + "loss": 2.8731, + "theoretical_loss": 3.8018641475310186, + "tokens_seen": 660602880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039418254764293, + "loss": 2.7511, + "theoretical_loss": 3.801825456411201, + "tokens_seen": 660668416 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040393179538615847, + "loss": 2.8653, + "theoretical_loss": 3.8017867702037424, + "tokens_seen": 660733952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040392176529588765, + "loss": 2.764, + "theoretical_loss": 3.801748088907531, + "tokens_seen": 660799488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040391173520561683, + "loss": 2.763, + "theoretical_loss": 3.8017094125214577, + "tokens_seen": 660865024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040390170511534607, + "loss": 2.9756, + "theoretical_loss": 3.8016707410444117, + "tokens_seen": 660930560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040389167502507525, + "loss": 2.8566, + "theoretical_loss": 3.8016320744752834, + "tokens_seen": 660996096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040388164493480443, + "loss": 2.8749, + "theoretical_loss": 3.8015934128129634, + "tokens_seen": 661061632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038716148445336, + "loss": 2.8313, + "theoretical_loss": 3.8015547560563423, + "tokens_seen": 661127168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040386158475426285, + "loss": 2.8092, + "theoretical_loss": 3.801516104204312, + "tokens_seen": 661192704 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403851554663992, + "loss": 2.9406, + "theoretical_loss": 3.801477457255764, + "tokens_seen": 661258240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038415245737212, + "loss": 2.7659, + "theoretical_loss": 3.8014388152095906, + "tokens_seen": 661323776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040383149448345034, + "loss": 2.9872, + "theoretical_loss": 3.8014001780646836, + "tokens_seen": 661389312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040382146439317957, + "loss": 2.9488, + "theoretical_loss": 3.801361545819936, + "tokens_seen": 661454848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040381143430290875, + "loss": 2.9806, + "theoretical_loss": 3.8013229184742414, + "tokens_seen": 661520384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040380140421263794, + "loss": 2.9426, + "theoretical_loss": 3.801284296026492, + "tokens_seen": 661585920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037913741223671, + "loss": 2.6688, + "theoretical_loss": 3.8012456784755835, + "tokens_seen": 661651456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037813440320963, + "loss": 2.8687, + "theoretical_loss": 3.801207065820409, + "tokens_seen": 661716992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037713139418255, + "loss": 2.9271, + "theoretical_loss": 3.8011684580598626, + "tokens_seen": 661782528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037612838515547, + "loss": 2.7706, + "theoretical_loss": 3.8011298551928405, + "tokens_seen": 661848064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 374068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0581512451171875, + "objective/train/theoretical_loss": 3.801091257218237, + "objective/train/tokens_used": 682373600, + "theoretical_loss": 3.801091257218237, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040375125376128384, + "loss": 2.7388, + "theoretical_loss": 3.801091257218237, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037412236710131, + "loss": 2.5943, + "theoretical_loss": 3.801052664134948, + "tokens_seen": 661979136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037311935807422, + "loss": 2.9245, + "theoretical_loss": 3.80101407594187, + "tokens_seen": 662044672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040372116349047144, + "loss": 2.7558, + "theoretical_loss": 3.8009754926378982, + "tokens_seen": 662110208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037111334002006, + "loss": 2.9066, + "theoretical_loss": 3.8009369142219303, + "tokens_seen": 662175744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037011033099298, + "loss": 2.6965, + "theoretical_loss": 3.8008983406928634, + "tokens_seen": 662241280 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403691073219659, + "loss": 2.8863, + "theoretical_loss": 3.800859772049595, + "tokens_seen": 662306816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036810431293882, + "loss": 2.7285, + "theoretical_loss": 3.8008212082910218, + "tokens_seen": 662372352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040367101303911734, + "loss": 2.8133, + "theoretical_loss": 3.8007826494160426, + "tokens_seen": 662437888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036609829488466, + "loss": 2.7862, + "theoretical_loss": 3.8007440954235565, + "tokens_seen": 662503424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036509528585757, + "loss": 2.635, + "theoretical_loss": 3.8007055463124617, + "tokens_seen": 662568960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040364092276830494, + "loss": 2.6339, + "theoretical_loss": 3.8006670020816578, + "tokens_seen": 662634496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036308926780341, + "loss": 2.8162, + "theoretical_loss": 3.800628462730044, + "tokens_seen": 662700032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036208625877633, + "loss": 2.8348, + "theoretical_loss": 3.8005899282565205, + "tokens_seen": 662765568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036108324974925, + "loss": 2.8832, + "theoretical_loss": 3.8005513986599873, + "tokens_seen": 662831104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040360080240722167, + "loss": 2.8673, + "theoretical_loss": 3.8005128739393452, + "tokens_seen": 662896640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040359077231695085, + "loss": 3.0328, + "theoretical_loss": 3.8004743540934953, + "tokens_seen": 662962176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035807422266801, + "loss": 2.6331, + "theoretical_loss": 3.8004358391213393, + "tokens_seen": 663027712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035707121364092, + "loss": 2.6823, + "theoretical_loss": 3.800397329021778, + "tokens_seen": 663093248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040356068204613844, + "loss": 2.7886, + "theoretical_loss": 3.8003588237937143, + "tokens_seen": 663158784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040355065195586757, + "loss": 3.0434, + "theoretical_loss": 3.8003203234360505, + "tokens_seen": 663224320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035406218655968, + "loss": 2.8486, + "theoretical_loss": 3.800281827947689, + "tokens_seen": 663289856 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403530591775326, + "loss": 2.8891, + "theoretical_loss": 3.800243337327533, + "tokens_seen": 663355392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040352056168505517, + "loss": 2.98, + "theoretical_loss": 3.8002048515744864, + "tokens_seen": 663420928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040351053159478435, + "loss": 3.0469, + "theoretical_loss": 3.800166370687453, + "tokens_seen": 663486464 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 374525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.942661762237549, + "objective/train/theoretical_loss": 3.8001278946653363, + "objective/train/tokens_used": 684012000, + "theoretical_loss": 3.8001278946653363, + "tokens_seen": 663552000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035005015045136, + "loss": 2.7616, + "theoretical_loss": 3.8001278946653363, + "tokens_seen": 663552000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034904714142427, + "loss": 2.6917, + "theoretical_loss": 3.8000894235070417, + "tokens_seen": 663617536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040348044132397195, + "loss": 2.8898, + "theoretical_loss": 3.800050957211474, + "tokens_seen": 663683072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034704112337011, + "loss": 2.6352, + "theoretical_loss": 3.800012495777538, + "tokens_seen": 663748608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034603811434303, + "loss": 2.9331, + "theoretical_loss": 3.7999740392041392, + "tokens_seen": 663814144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034503510531595, + "loss": 2.8874, + "theoretical_loss": 3.799935587490185, + "tokens_seen": 663879680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040344032096288867, + "loss": 3.0548, + "theoretical_loss": 3.7998971406345796, + "tokens_seen": 663945216 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040343029087261785, + "loss": 2.5845, + "theoretical_loss": 3.7998586986362315, + "tokens_seen": 664010752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040342026078234703, + "loss": 2.8137, + "theoretical_loss": 3.799820261494047, + "tokens_seen": 664076288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034102306920762, + "loss": 2.8299, + "theoretical_loss": 3.7997818292069327, + "tokens_seen": 664141824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040340020060180545, + "loss": 2.8644, + "theoretical_loss": 3.7997434017737985, + "tokens_seen": 664207360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033901705115346, + "loss": 2.6154, + "theoretical_loss": 3.7997049791935504, + "tokens_seen": 664272896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033801404212638, + "loss": 2.739, + "theoretical_loss": 3.7996665614650977, + "tokens_seen": 664338432 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403370110330993, + "loss": 2.6332, + "theoretical_loss": 3.799628148587349, + "tokens_seen": 664403968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033600802407222, + "loss": 2.6486, + "theoretical_loss": 3.799589740559214, + "tokens_seen": 664469504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040335005015045136, + "loss": 2.8741, + "theoretical_loss": 3.7995513373796017, + "tokens_seen": 664535040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040334002006018054, + "loss": 3.0168, + "theoretical_loss": 3.799512939047422, + "tokens_seen": 664600576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033299899699097, + "loss": 2.946, + "theoretical_loss": 3.7994745455615857, + "tokens_seen": 664666112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040331995987963895, + "loss": 2.8055, + "theoretical_loss": 3.7994361569210024, + "tokens_seen": 664731648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033099297893681, + "loss": 2.9599, + "theoretical_loss": 3.799397773124584, + "tokens_seen": 664797184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032998996990973, + "loss": 2.7457, + "theoretical_loss": 3.7993593941712405, + "tokens_seen": 664862720 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040328986960882644, + "loss": 2.9906, + "theoretical_loss": 3.7993210200598853, + "tokens_seen": 664928256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032798395185557, + "loss": 2.7725, + "theoretical_loss": 3.7992826507894284, + "tokens_seen": 664993792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040326980942828486, + "loss": 2.564, + "theoretical_loss": 3.7992442863587836, + "tokens_seen": 665059328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040325977933801404, + "loss": 2.7479, + "theoretical_loss": 3.7992059267668634, + "tokens_seen": 665124864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 375614, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8573155403137207, + "objective/train/theoretical_loss": 3.79916757201258, + "objective/train/tokens_used": 685650400, + "theoretical_loss": 3.79916757201258, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032497492477432, + "loss": 2.8858, + "theoretical_loss": 3.79916757201258, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032397191574724, + "loss": 3.0415, + "theoretical_loss": 3.799129222094848, + "tokens_seen": 665255936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032296890672016, + "loss": 3.0551, + "theoretical_loss": 3.7990908770125804, + "tokens_seen": 665321472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032196589769308, + "loss": 2.8895, + "theoretical_loss": 3.799052536764691, + "tokens_seen": 665387008 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040320962888665995, + "loss": 2.7187, + "theoretical_loss": 3.7990142013500945, + "tokens_seen": 665452544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031995987963892, + "loss": 2.8924, + "theoretical_loss": 3.7989758707677064, + "tokens_seen": 665518080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040318956870611836, + "loss": 2.8798, + "theoretical_loss": 3.7989375450164413, + "tokens_seen": 665583616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040317953861584754, + "loss": 2.915, + "theoretical_loss": 3.798899224095214, + "tokens_seen": 665649152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031695085255767, + "loss": 2.8973, + "theoretical_loss": 3.798860908002941, + "tokens_seen": 665714688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031594784353059, + "loss": 2.725, + "theoretical_loss": 3.798822596738539, + "tokens_seen": 665780224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040314944834503514, + "loss": 2.9635, + "theoretical_loss": 3.7987842903009237, + "tokens_seen": 665845760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031394182547643, + "loss": 2.8181, + "theoretical_loss": 3.7987459886890127, + "tokens_seen": 665911296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031293881644935, + "loss": 2.9355, + "theoretical_loss": 3.798707691901722, + "tokens_seen": 665976832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031193580742227, + "loss": 2.6325, + "theoretical_loss": 3.798669399937971, + "tokens_seen": 666042368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040310932798395187, + "loss": 2.7554, + "theoretical_loss": 3.7986311127966763, + "tokens_seen": 666107904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040309929789368105, + "loss": 2.917, + "theoretical_loss": 3.798592830476757, + "tokens_seen": 666173440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030892678034103, + "loss": 2.6412, + "theoretical_loss": 3.79855455297713, + "tokens_seen": 666238976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030792377131394, + "loss": 2.8717, + "theoretical_loss": 3.7985162802967167, + "tokens_seen": 666304512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040306920762286864, + "loss": 2.7658, + "theoretical_loss": 3.798478012434435, + "tokens_seen": 666370048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040305917753259777, + "loss": 3.1829, + "theoretical_loss": 3.7984397493892055, + "tokens_seen": 666435584 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403049147442327, + "loss": 2.9596, + "theoretical_loss": 3.798401491159947, + "tokens_seen": 666501120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030391173520562, + "loss": 2.9012, + "theoretical_loss": 3.7983632377455807, + "tokens_seen": 666566656 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040302908726178537, + "loss": 2.7809, + "theoretical_loss": 3.7983249891450273, + "tokens_seen": 666632192 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040301905717151455, + "loss": 2.761, + "theoretical_loss": 3.7982867453572076, + "tokens_seen": 666697728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030090270812438, + "loss": 2.7542, + "theoretical_loss": 3.798248506381043, + "tokens_seen": 666763264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 376311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.005213975906372, + "objective/train/theoretical_loss": 3.7982102722154556, + "objective/train/tokens_used": 687288800, + "theoretical_loss": 3.7982102722154556, + "tokens_seen": 666828800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029989969909729, + "loss": 2.8448, + "theoretical_loss": 3.7982102722154556, + "tokens_seen": 666828800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040298896690070215, + "loss": 2.6927, + "theoretical_loss": 3.7981720428593677, + "tokens_seen": 666894336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029789368104313, + "loss": 2.7377, + "theoretical_loss": 3.798133818311701, + "tokens_seen": 666959872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029689067201605, + "loss": 2.5689, + "theoretical_loss": 3.798095598571379, + "tokens_seen": 667025408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029588766298897, + "loss": 2.8091, + "theoretical_loss": 3.798057383637324, + "tokens_seen": 667090944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040294884653961887, + "loss": 2.9873, + "theoretical_loss": 3.798019173508461, + "tokens_seen": 667156480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040293881644934805, + "loss": 2.8216, + "theoretical_loss": 3.7979809681837127, + "tokens_seen": 667222016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040292878635907723, + "loss": 2.8597, + "theoretical_loss": 3.7979427676620032, + "tokens_seen": 667287552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029187562688064, + "loss": 2.8129, + "theoretical_loss": 3.797904571942258, + "tokens_seen": 667353088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040290872617853565, + "loss": 2.7172, + "theoretical_loss": 3.797866381023401, + "tokens_seen": 667418624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028986960882648, + "loss": 2.6435, + "theoretical_loss": 3.7978281949043575, + "tokens_seen": 667484160 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402888665997994, + "loss": 2.6663, + "theoretical_loss": 3.797790013584054, + "tokens_seen": 667549696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028786359077232, + "loss": 2.9507, + "theoretical_loss": 3.7977518370614156, + "tokens_seen": 667615232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028686058174524, + "loss": 2.8432, + "theoretical_loss": 3.797713665335369, + "tokens_seen": 667680768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040285857572718156, + "loss": 2.7833, + "theoretical_loss": 3.797675498404841, + "tokens_seen": 667746304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040284854563691074, + "loss": 2.8119, + "theoretical_loss": 3.7976373362687577, + "tokens_seen": 667811840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028385155466399, + "loss": 2.8527, + "theoretical_loss": 3.7975991789260473, + "tokens_seen": 667877376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040282848545636915, + "loss": 2.8808, + "theoretical_loss": 3.7975610263756368, + "tokens_seen": 667942912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028184553660983, + "loss": 2.6857, + "theoretical_loss": 3.797522878616455, + "tokens_seen": 668008448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028084252758275, + "loss": 2.9767, + "theoretical_loss": 3.7974847356474295, + "tokens_seen": 668073984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040279839518555664, + "loss": 2.8602, + "theoretical_loss": 3.797446597467489, + "tokens_seen": 668139520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027883650952859, + "loss": 2.8874, + "theoretical_loss": 3.797408464075563, + "tokens_seen": 668205056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040277833500501506, + "loss": 2.6017, + "theoretical_loss": 3.797370335470581, + "tokens_seen": 668270592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040276830491474424, + "loss": 2.5136, + "theoretical_loss": 3.7973322116514723, + "tokens_seen": 668336128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027582748244734, + "loss": 2.704, + "theoretical_loss": 3.797294092617167, + "tokens_seen": 668401664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 377619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8243908882141113, + "objective/train/theoretical_loss": 3.7972559783665965, + "objective/train/tokens_used": 688927200, + "theoretical_loss": 3.7972559783665965, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027482447342026, + "loss": 2.5602, + "theoretical_loss": 3.7972559783665965, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027382146439318, + "loss": 2.8683, + "theoretical_loss": 3.7972178688986897, + "tokens_seen": 668532736 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402728184553661, + "loss": 2.5877, + "theoretical_loss": 3.7971797642123786, + "tokens_seen": 668598272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040271815446339015, + "loss": 2.9583, + "theoretical_loss": 3.7971416643065954, + "tokens_seen": 668663808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027081243731194, + "loss": 2.7348, + "theoretical_loss": 3.797103569180271, + "tokens_seen": 668729344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040269809428284856, + "loss": 2.7532, + "theoretical_loss": 3.7970654788323377, + "tokens_seen": 668794880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040268806419257774, + "loss": 2.9621, + "theoretical_loss": 3.797027393261728, + "tokens_seen": 668860416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026780341023069, + "loss": 2.8518, + "theoretical_loss": 3.7969893124673746, + "tokens_seen": 668925952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026680040120361, + "loss": 2.7747, + "theoretical_loss": 3.7969512364482116, + "tokens_seen": 668991488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026579739217653, + "loss": 2.7672, + "theoretical_loss": 3.796913165203171, + "tokens_seen": 669057024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026479438314945, + "loss": 2.896, + "theoretical_loss": 3.7968750987311877, + "tokens_seen": 669122560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040263791374122365, + "loss": 2.9705, + "theoretical_loss": 3.796837037031196, + "tokens_seen": 669188096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040263791374122365, + "loss": 3.0305, + "theoretical_loss": 3.796798980102129, + "tokens_seen": 669253632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026278836509529, + "loss": 2.8599, + "theoretical_loss": 3.7967609279429233, + "tokens_seen": 669319168 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402617853560682, + "loss": 2.8396, + "theoretical_loss": 3.7967228805525135, + "tokens_seen": 669384704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040260782347041125, + "loss": 2.7468, + "theoretical_loss": 3.796684837929835, + "tokens_seen": 669450240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040259779338014043, + "loss": 2.9226, + "theoretical_loss": 3.796646800073824, + "tokens_seen": 669515776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025877632898696, + "loss": 2.8581, + "theoretical_loss": 3.796608766983416, + "tokens_seen": 669581312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025777331995988, + "loss": 2.9392, + "theoretical_loss": 3.7965707386575485, + "tokens_seen": 669646848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040256770310932797, + "loss": 2.8257, + "theoretical_loss": 3.796532715095158, + "tokens_seen": 669712384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040255767301905715, + "loss": 2.7459, + "theoretical_loss": 3.7964946962951815, + "tokens_seen": 669777920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025476429287864, + "loss": 3.0115, + "theoretical_loss": 3.7964566822565575, + "tokens_seen": 669843456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025376128385155, + "loss": 2.7659, + "theoretical_loss": 3.796418672978223, + "tokens_seen": 669908992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040252758274824475, + "loss": 2.7765, + "theoretical_loss": 3.7963806684591166, + "tokens_seen": 669974528 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040251755265797393, + "loss": 2.9028, + "theoretical_loss": 3.7963426686981774, + "tokens_seen": 670040064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 378282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.932396650314331, + "objective/train/theoretical_loss": 3.796304673694344, + "objective/train/tokens_used": 690565600, + "theoretical_loss": 3.796304673694344, + "tokens_seen": 670105600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025075225677031, + "loss": 2.9771, + "theoretical_loss": 3.796304673694344, + "tokens_seen": 670105600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024974924774323, + "loss": 2.7668, + "theoretical_loss": 3.7962666834465555, + "tokens_seen": 670171136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024874623871615, + "loss": 2.634, + "theoretical_loss": 3.796228697953752, + "tokens_seen": 670236672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040247743229689066, + "loss": 2.9584, + "theoretical_loss": 3.7961907172148726, + "tokens_seen": 670302208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024674022066199, + "loss": 2.8543, + "theoretical_loss": 3.7961527412288585, + "tokens_seen": 670367744 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402457372116349, + "loss": 3.0343, + "theoretical_loss": 3.7961147699946505, + "tokens_seen": 670433280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040244734202607825, + "loss": 2.8191, + "theoretical_loss": 3.796076803511189, + "tokens_seen": 670498816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024373119358074, + "loss": 2.7031, + "theoretical_loss": 3.7960388417774156, + "tokens_seen": 670564352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024272818455366, + "loss": 3.0333, + "theoretical_loss": 3.796000884792272, + "tokens_seen": 670629888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024172517552658, + "loss": 2.9055, + "theoretical_loss": 3.7959629325547004, + "tokens_seen": 670695424 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402407221664995, + "loss": 3.193, + "theoretical_loss": 3.7959249850636425, + "tokens_seen": 670760960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023971915747242, + "loss": 2.7088, + "theoretical_loss": 3.795887042318042, + "tokens_seen": 670826496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023871614844534, + "loss": 2.6832, + "theoretical_loss": 3.7958491043168414, + "tokens_seen": 670892032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023771313941826, + "loss": 2.6732, + "theoretical_loss": 3.7958111710589844, + "tokens_seen": 670957568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040236710130391176, + "loss": 2.7021, + "theoretical_loss": 3.7957732425434143, + "tokens_seen": 671023104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040235707121364094, + "loss": 2.4503, + "theoretical_loss": 3.7957353187690748, + "tokens_seen": 671088640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023470411233701, + "loss": 2.8226, + "theoretical_loss": 3.7956973997349115, + "tokens_seen": 671154176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040233701103309935, + "loss": 3.0277, + "theoretical_loss": 3.7956594854398684, + "tokens_seen": 671219712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023269809428285, + "loss": 2.7509, + "theoretical_loss": 3.7956215758828904, + "tokens_seen": 671285248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023169508525577, + "loss": 2.7952, + "theoretical_loss": 3.795583671062923, + "tokens_seen": 671350784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040230692076228684, + "loss": 2.8198, + "theoretical_loss": 3.7955457709789124, + "tokens_seen": 671416320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022968906720161, + "loss": 2.8985, + "theoretical_loss": 3.795507875629805, + "tokens_seen": 671481856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040228686058174526, + "loss": 2.5066, + "theoretical_loss": 3.7954699850145457, + "tokens_seen": 671547392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040227683049147444, + "loss": 2.6576, + "theoretical_loss": 3.795432099132083, + "tokens_seen": 671612928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022668004012036, + "loss": 2.6896, + "theoretical_loss": 3.7953942179813627, + "tokens_seen": 671678464 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 379317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.70872163772583, + "objective/train/theoretical_loss": 3.7953563415613325, + "objective/train/tokens_used": 692204000, + "theoretical_loss": 3.7953563415613325, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022567703109328, + "loss": 2.8489, + "theoretical_loss": 3.7953563415613325, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402246740220662, + "loss": 2.6866, + "theoretical_loss": 3.795318469870941, + "tokens_seen": 671809536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022367101303912, + "loss": 3.0096, + "theoretical_loss": 3.795280602909135, + "tokens_seen": 671875072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040222668004012035, + "loss": 2.771, + "theoretical_loss": 3.7952427406748646, + "tokens_seen": 671940608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022166499498496, + "loss": 2.5575, + "theoretical_loss": 3.7952048831670773, + "tokens_seen": 672006144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040220661985957876, + "loss": 2.7143, + "theoretical_loss": 3.795167030384722, + "tokens_seen": 672071680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040219658976930794, + "loss": 2.7934, + "theoretical_loss": 3.7951291823267495, + "tokens_seen": 672137216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021865596790371, + "loss": 2.986, + "theoretical_loss": 3.7950913389921084, + "tokens_seen": 672202752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021765295887663, + "loss": 2.8392, + "theoretical_loss": 3.7950535003797494, + "tokens_seen": 672268288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021664994984955, + "loss": 2.7923, + "theoretical_loss": 3.795015666488623, + "tokens_seen": 672333824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021564694082247, + "loss": 2.7625, + "theoretical_loss": 3.7949778373176795, + "tokens_seen": 672399360 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040214643931795385, + "loss": 2.8883, + "theoretical_loss": 3.7949400128658706, + "tokens_seen": 672464896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021364092276831, + "loss": 2.8509, + "theoretical_loss": 3.7949021931321476, + "tokens_seen": 672530432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021263791374122, + "loss": 2.662, + "theoretical_loss": 3.794864378115462, + "tokens_seen": 672595968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040211634904714145, + "loss": 2.6493, + "theoretical_loss": 3.7948265678147663, + "tokens_seen": 672661504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040210631895687063, + "loss": 2.8812, + "theoretical_loss": 3.7947887622290133, + "tokens_seen": 672727040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020962888665998, + "loss": 2.9279, + "theoretical_loss": 3.7947509613571553, + "tokens_seen": 672792576 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402086258776329, + "loss": 2.8662, + "theoretical_loss": 3.7947131651981447, + "tokens_seen": 672858112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040207622868605817, + "loss": 2.9209, + "theoretical_loss": 3.7946753737509367, + "tokens_seen": 672923648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040206619859578735, + "loss": 2.86, + "theoretical_loss": 3.794637587014484, + "tokens_seen": 672989184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020561685055166, + "loss": 2.9311, + "theoretical_loss": 3.7945998049877403, + "tokens_seen": 673054720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020461384152457, + "loss": 2.7814, + "theoretical_loss": 3.794562027669661, + "tokens_seen": 673120256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040203610832497495, + "loss": 3.022, + "theoretical_loss": 3.7945242550592013, + "tokens_seen": 673185792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040202607823470413, + "loss": 2.8168, + "theoretical_loss": 3.7944864871553152, + "tokens_seen": 673251328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020160481444333, + "loss": 2.8123, + "theoretical_loss": 3.794448723956959, + "tokens_seen": 673316864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 379824, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8459346294403076, + "objective/train/theoretical_loss": 3.794410965463088, + "objective/train/tokens_used": 693842400, + "theoretical_loss": 3.794410965463088, + "tokens_seen": 673382400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020060180541625, + "loss": 2.9922, + "theoretical_loss": 3.794410965463088, + "tokens_seen": 673382400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019959879638917, + "loss": 2.5647, + "theoretical_loss": 3.7943732116726583, + "tokens_seen": 673447936 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040198595787362086, + "loss": 2.6541, + "theoretical_loss": 3.794335462584627, + "tokens_seen": 673513472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019759277833501, + "loss": 2.7915, + "theoretical_loss": 3.7942977181979503, + "tokens_seen": 673579008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019658976930792, + "loss": 2.7506, + "theoretical_loss": 3.794259978511586, + "tokens_seen": 673644544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040195586760280845, + "loss": 2.8794, + "theoretical_loss": 3.794222243524491, + "tokens_seen": 673710080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019458375125376, + "loss": 2.9101, + "theoretical_loss": 3.7941845132356233, + "tokens_seen": 673775616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019358074222668, + "loss": 3.0697, + "theoretical_loss": 3.7941467876439408, + "tokens_seen": 673841152 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401925777331996, + "loss": 3.0465, + "theoretical_loss": 3.7941090667484025, + "tokens_seen": 673906688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019157472417252, + "loss": 2.7341, + "theoretical_loss": 3.794071350547967, + "tokens_seen": 673972224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040190571715145436, + "loss": 2.8905, + "theoretical_loss": 3.794033639041593, + "tokens_seen": 674037760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018956870611836, + "loss": 2.7838, + "theoretical_loss": 3.793995932228241, + "tokens_seen": 674103296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018856569709127, + "loss": 2.8153, + "theoretical_loss": 3.7939582301068695, + "tokens_seen": 674168832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040187562688064196, + "loss": 2.647, + "theoretical_loss": 3.79392053267644, + "tokens_seen": 674234368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018655967903711, + "loss": 2.6909, + "theoretical_loss": 3.7938828399359115, + "tokens_seen": 674299904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018555667001003, + "loss": 3.1, + "theoretical_loss": 3.7938451518842458, + "tokens_seen": 674365440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018455366098295, + "loss": 2.9233, + "theoretical_loss": 3.793807468520404, + "tokens_seen": 674430976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018355065195587, + "loss": 2.8234, + "theoretical_loss": 3.7937697898433473, + "tokens_seen": 674496512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040182547642928786, + "loss": 2.998, + "theoretical_loss": 3.7937321158520376, + "tokens_seen": 674562048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040181544633901704, + "loss": 2.9544, + "theoretical_loss": 3.7936944465454374, + "tokens_seen": 674627584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018054162487462, + "loss": 2.7967, + "theoretical_loss": 3.7936567819225084, + "tokens_seen": 674693120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040179538615847546, + "loss": 2.8876, + "theoretical_loss": 3.7936191219822133, + "tokens_seen": 674758656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017853560682046, + "loss": 2.907, + "theoretical_loss": 3.7935814667235164, + "tokens_seen": 674824192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017753259779338, + "loss": 2.9187, + "theoretical_loss": 3.79354381614538, + "tokens_seen": 674889728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040176529588766295, + "loss": 2.9948, + "theoretical_loss": 3.793506170246768, + "tokens_seen": 674955264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 381014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8223540782928467, + "objective/train/theoretical_loss": 3.7934685290266454, + "objective/train/tokens_used": 695480800, + "theoretical_loss": 3.7934685290266454, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040176529588766295, + "loss": 2.8171, + "theoretical_loss": 3.7934685290266454, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017552657973922, + "loss": 2.9283, + "theoretical_loss": 3.7934308924839755, + "tokens_seen": 675086336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040174523570712137, + "loss": 2.9153, + "theoretical_loss": 3.793393260617724, + "tokens_seen": 675151872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040173520561685055, + "loss": 2.7576, + "theoretical_loss": 3.7933556334268554, + "tokens_seen": 675217408 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040172517552657973, + "loss": 2.7632, + "theoretical_loss": 3.793318010910335, + "tokens_seen": 675282944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040171514543630896, + "loss": 2.6694, + "theoretical_loss": 3.7932803930671297, + "tokens_seen": 675348480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017051153460381, + "loss": 2.9775, + "theoretical_loss": 3.7932427798962043, + "tokens_seen": 675414016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016950852557673, + "loss": 2.9839, + "theoretical_loss": 3.793205171396526, + "tokens_seen": 675479552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040168505516549645, + "loss": 2.7652, + "theoretical_loss": 3.7931675675670604, + "tokens_seen": 675545088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016750250752257, + "loss": 2.7479, + "theoretical_loss": 3.7931299684067756, + "tokens_seen": 675610624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040166499498495487, + "loss": 2.564, + "theoretical_loss": 3.7930923739146394, + "tokens_seen": 675676160 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040165496489468405, + "loss": 2.5691, + "theoretical_loss": 3.7930547840896187, + "tokens_seen": 675741696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016449348044133, + "loss": 2.9134, + "theoretical_loss": 3.7930171989306816, + "tokens_seen": 675807232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016349047141424, + "loss": 2.7036, + "theoretical_loss": 3.7929796184367968, + "tokens_seen": 675872768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040162487462387165, + "loss": 2.6151, + "theoretical_loss": 3.7929420426069327, + "tokens_seen": 675938304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040161484453360083, + "loss": 2.7893, + "theoretical_loss": 3.7929044714400586, + "tokens_seen": 676003840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040160481444333, + "loss": 2.6284, + "theoretical_loss": 3.792866904935144, + "tokens_seen": 676069376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015947843530592, + "loss": 2.7396, + "theoretical_loss": 3.792829343091158, + "tokens_seen": 676134912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040158475426278837, + "loss": 2.8118, + "theoretical_loss": 3.7927917859070712, + "tokens_seen": 676200448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040157472417251755, + "loss": 2.774, + "theoretical_loss": 3.792754233381854, + "tokens_seen": 676265984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015646940822468, + "loss": 3.0124, + "theoretical_loss": 3.7927166855144763, + "tokens_seen": 676331520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015546639919759, + "loss": 2.5057, + "theoretical_loss": 3.7926791423039097, + "tokens_seen": 676397056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040154463390170515, + "loss": 2.8492, + "theoretical_loss": 3.792641603749126, + "tokens_seen": 676462592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040153460381143433, + "loss": 2.8639, + "theoretical_loss": 3.7926040698490957, + "tokens_seen": 676528128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015245737211635, + "loss": 3.042, + "theoretical_loss": 3.7925665406027917, + "tokens_seen": 676593664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 382043, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4060447216033936, + "objective/train/theoretical_loss": 3.792529016009186, + "objective/train/tokens_used": 697119200, + "theoretical_loss": 3.792529016009186, + "tokens_seen": 676659200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015145436308927, + "loss": 2.8706, + "theoretical_loss": 3.792529016009186, + "tokens_seen": 676659200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015045135406219, + "loss": 2.7768, + "theoretical_loss": 3.7924914960672513, + "tokens_seen": 676724736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040149448345035106, + "loss": 2.817, + "theoretical_loss": 3.792453980775961, + "tokens_seen": 676790272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014844533600803, + "loss": 2.7802, + "theoretical_loss": 3.7924164701342873, + "tokens_seen": 676855808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014744232698094, + "loss": 2.9134, + "theoretical_loss": 3.7923789641412045, + "tokens_seen": 676921344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040146439317953865, + "loss": 2.8138, + "theoretical_loss": 3.7923414627956866, + "tokens_seen": 676986880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014543630892678, + "loss": 2.6232, + "theoretical_loss": 3.792303966096708, + "tokens_seen": 677052416 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401444332998997, + "loss": 2.9834, + "theoretical_loss": 3.792266474043243, + "tokens_seen": 677117952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014343029087262, + "loss": 2.8613, + "theoretical_loss": 3.7922289866342664, + "tokens_seen": 677183488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014242728184554, + "loss": 2.7659, + "theoretical_loss": 3.7921915038687537, + "tokens_seen": 677249024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040141424272818456, + "loss": 2.8298, + "theoretical_loss": 3.7921540257456803, + "tokens_seen": 677314560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014042126379138, + "loss": 2.7942, + "theoretical_loss": 3.7921165522640226, + "tokens_seen": 677380096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013941825476429, + "loss": 2.4966, + "theoretical_loss": 3.792079083422756, + "tokens_seen": 677445632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040138415245737216, + "loss": 2.7792, + "theoretical_loss": 3.7920416192208575, + "tokens_seen": 677511168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013741223671013, + "loss": 2.6478, + "theoretical_loss": 3.7920041596573038, + "tokens_seen": 677576704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013640922768305, + "loss": 2.6984, + "theoretical_loss": 3.7919667047310726, + "tokens_seen": 677642240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013540621865597, + "loss": 2.6497, + "theoretical_loss": 3.7919292544411407, + "tokens_seen": 677707776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013440320962889, + "loss": 2.7696, + "theoretical_loss": 3.791891808786487, + "tokens_seen": 677773312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040133400200601806, + "loss": 2.8313, + "theoretical_loss": 3.791854367766088, + "tokens_seen": 677838848 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040132397191574724, + "loss": 2.9008, + "theoretical_loss": 3.7918169313789236, + "tokens_seen": 677904384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013139418254764, + "loss": 2.7586, + "theoretical_loss": 3.7917794996239724, + "tokens_seen": 677969920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040130391173520566, + "loss": 2.8737, + "theoretical_loss": 3.791742072500213, + "tokens_seen": 678035456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012938816449348, + "loss": 2.9405, + "theoretical_loss": 3.7917046500066256, + "tokens_seen": 678100992 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401283851554664, + "loss": 2.7992, + "theoretical_loss": 3.7916672321421894, + "tokens_seen": 678166528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040127382146439315, + "loss": 2.8993, + "theoretical_loss": 3.791629818905884, + "tokens_seen": 678232064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 382695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.529493808746338, + "objective/train/theoretical_loss": 3.7915924102966914, + "objective/train/tokens_used": 698757600, + "theoretical_loss": 3.7915924102966914, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012637913741224, + "loss": 2.7103, + "theoretical_loss": 3.7915924102966914, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040125376128385157, + "loss": 2.852, + "theoretical_loss": 3.7915550063135908, + "tokens_seen": 678363136 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040124373119358075, + "loss": 3.0078, + "theoretical_loss": 3.7915176069555643, + "tokens_seen": 678428672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040123370110330993, + "loss": 2.9411, + "theoretical_loss": 3.791480212221593, + "tokens_seen": 678494208 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040122367101303916, + "loss": 2.9414, + "theoretical_loss": 3.7914428221106586, + "tokens_seen": 678559744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012136409227683, + "loss": 2.8493, + "theoretical_loss": 3.791405436621743, + "tokens_seen": 678625280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012036108324975, + "loss": 2.5601, + "theoretical_loss": 3.7913680557538285, + "tokens_seen": 678690816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040119358074222665, + "loss": 2.6255, + "theoretical_loss": 3.791330679505898, + "tokens_seen": 678756352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011835506519559, + "loss": 2.9282, + "theoretical_loss": 3.791293307876935, + "tokens_seen": 678821888 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040117352056168507, + "loss": 2.784, + "theoretical_loss": 3.7912559408659217, + "tokens_seen": 678887424 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040116349047141425, + "loss": 2.7854, + "theoretical_loss": 3.7912185784718435, + "tokens_seen": 678952960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040115346038114343, + "loss": 2.6421, + "theoretical_loss": 3.7911812206936824, + "tokens_seen": 679018496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011434302908726, + "loss": 2.9031, + "theoretical_loss": 3.7911438675304234, + "tokens_seen": 679084032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011334002006018, + "loss": 2.8137, + "theoretical_loss": 3.791106518981052, + "tokens_seen": 679149568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040112337011033103, + "loss": 3.0802, + "theoretical_loss": 3.7910691750445515, + "tokens_seen": 679215104 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040111334002006016, + "loss": 2.664, + "theoretical_loss": 3.7910318357199086, + "tokens_seen": 679280640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011033099297894, + "loss": 2.9432, + "theoretical_loss": 3.790994501006109, + "tokens_seen": 679346176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010932798395185, + "loss": 2.9391, + "theoretical_loss": 3.7909571709021375, + "tokens_seen": 679411712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040108324974924775, + "loss": 2.8087, + "theoretical_loss": 3.7909198454069806, + "tokens_seen": 679477248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040107321965897693, + "loss": 2.7766, + "theoretical_loss": 3.790882524519626, + "tokens_seen": 679542784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010631895687061, + "loss": 2.8428, + "theoretical_loss": 3.7908452082390585, + "tokens_seen": 679608320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010531594784353, + "loss": 2.7802, + "theoretical_loss": 3.7908078965642673, + "tokens_seen": 679673856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040104312938816453, + "loss": 2.659, + "theoretical_loss": 3.7907705894942385, + "tokens_seen": 679739392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040103309929789366, + "loss": 2.6857, + "theoretical_loss": 3.790733287027961, + "tokens_seen": 679804928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010230692076229, + "loss": 2.7388, + "theoretical_loss": 3.790695989164422, + "tokens_seen": 679870464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 383786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7080867290496826, + "objective/train/theoretical_loss": 3.7906586959026107, + "objective/train/tokens_used": 700396000, + "theoretical_loss": 3.7906586959026107, + "tokens_seen": 679936000 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401013039117352, + "loss": 2.8253, + "theoretical_loss": 3.7906586959026107, + "tokens_seen": 679936000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040100300902708126, + "loss": 3.1717, + "theoretical_loss": 3.7906214072415154, + "tokens_seen": 680001536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040099297893681044, + "loss": 2.9097, + "theoretical_loss": 3.7905841231801256, + "tokens_seen": 680067072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009829488465396, + "loss": 2.8925, + "theoretical_loss": 3.790546843717431, + "tokens_seen": 680132608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009729187562688, + "loss": 2.6975, + "theoretical_loss": 3.79050956885242, + "tokens_seen": 680198144 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400962888665998, + "loss": 2.6732, + "theoretical_loss": 3.790472298584084, + "tokens_seen": 680263680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040095285857572716, + "loss": 2.906, + "theoretical_loss": 3.7904350329114127, + "tokens_seen": 680329216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009428284854564, + "loss": 2.7083, + "theoretical_loss": 3.7903977718333977, + "tokens_seen": 680394752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009327983951855, + "loss": 2.8424, + "theoretical_loss": 3.790360515349029, + "tokens_seen": 680460288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040092276830491476, + "loss": 2.9744, + "theoretical_loss": 3.7903232634572985, + "tokens_seen": 680525824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009127382146439, + "loss": 2.8977, + "theoretical_loss": 3.7902860161571974, + "tokens_seen": 680591360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009027081243731, + "loss": 2.7069, + "theoretical_loss": 3.7902487734477184, + "tokens_seen": 680656896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040089267803410236, + "loss": 2.8793, + "theoretical_loss": 3.790211535327853, + "tokens_seen": 680722432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008826479438315, + "loss": 2.6679, + "theoretical_loss": 3.7901743017965943, + "tokens_seen": 680787968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008726178535607, + "loss": 2.8633, + "theoretical_loss": 3.7901370728529353, + "tokens_seen": 680853504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008625877632899, + "loss": 2.8139, + "theoretical_loss": 3.790099848495869, + "tokens_seen": 680919040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008525576730191, + "loss": 2.8153, + "theoretical_loss": 3.790062628724389, + "tokens_seen": 680984576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040084252758274826, + "loss": 2.7675, + "theoretical_loss": 3.790025413537489, + "tokens_seen": 681050112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040083249749247744, + "loss": 2.9715, + "theoretical_loss": 3.789988202934164, + "tokens_seen": 681115648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008224674022066, + "loss": 2.936, + "theoretical_loss": 3.789950996913408, + "tokens_seen": 681181184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040081243731193586, + "loss": 2.655, + "theoretical_loss": 3.7899137954742153, + "tokens_seen": 681246720 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400802407221665, + "loss": 3.0181, + "theoretical_loss": 3.7898765986155816, + "tokens_seen": 681312256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007923771313942, + "loss": 2.9044, + "theoretical_loss": 3.789839406336503, + "tokens_seen": 681377792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040078234704112335, + "loss": 2.8241, + "theoretical_loss": 3.789802218635974, + "tokens_seen": 681443328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007723169508526, + "loss": 2.7947, + "theoretical_loss": 3.7897650355129917, + "tokens_seen": 681508864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 384361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9259161949157715, + "objective/train/theoretical_loss": 3.789727856966552, + "objective/train/tokens_used": 702034400, + "theoretical_loss": 3.789727856966552, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040076228686058177, + "loss": 2.7609, + "theoretical_loss": 3.789727856966552, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040075225677031095, + "loss": 2.7081, + "theoretical_loss": 3.7896906829956523, + "tokens_seen": 681639936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040074222668004013, + "loss": 2.9535, + "theoretical_loss": 3.789653513599289, + "tokens_seen": 681705472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040073219658976936, + "loss": 2.7223, + "theoretical_loss": 3.7896163487764594, + "tokens_seen": 681771008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007221664994985, + "loss": 2.8175, + "theoretical_loss": 3.7895791885261616, + "tokens_seen": 681836544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007121364092277, + "loss": 2.9393, + "theoretical_loss": 3.789542032847393, + "tokens_seen": 681902080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040070210631895685, + "loss": 2.6259, + "theoretical_loss": 3.7895048817391532, + "tokens_seen": 681967616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006920762286861, + "loss": 2.7234, + "theoretical_loss": 3.78946773520044, + "tokens_seen": 682033152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040068204613841527, + "loss": 2.882, + "theoretical_loss": 3.789430593230252, + "tokens_seen": 682098688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040067201604814445, + "loss": 2.5952, + "theoretical_loss": 3.7893934558275886, + "tokens_seen": 682164224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040066198595787363, + "loss": 2.914, + "theoretical_loss": 3.78935632299145, + "tokens_seen": 682229760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006519558676028, + "loss": 2.7516, + "theoretical_loss": 3.789319194720836, + "tokens_seen": 682295296 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400641925777332, + "loss": 2.6412, + "theoretical_loss": 3.789282071014746, + "tokens_seen": 682360832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040063189568706123, + "loss": 2.8659, + "theoretical_loss": 3.789244951872181, + "tokens_seen": 682426368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040062186559679036, + "loss": 2.6498, + "theoretical_loss": 3.7892078372921425, + "tokens_seen": 682491904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006118355065196, + "loss": 2.8014, + "theoretical_loss": 3.789170727273631, + "tokens_seen": 682557440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006018054162487, + "loss": 2.7229, + "theoretical_loss": 3.789133621815648, + "tokens_seen": 682622976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040059177532597795, + "loss": 2.7762, + "theoretical_loss": 3.7890965209171954, + "tokens_seen": 682688512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040058174523570713, + "loss": 2.7451, + "theoretical_loss": 3.7890594245772746, + "tokens_seen": 682754048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005717151454363, + "loss": 2.9726, + "theoretical_loss": 3.7890223327948895, + "tokens_seen": 682819584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005616850551655, + "loss": 2.6521, + "theoretical_loss": 3.788985245569042, + "tokens_seen": 682885120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040055165496489473, + "loss": 2.8672, + "theoretical_loss": 3.788948162898735, + "tokens_seen": 682950656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040054162487462386, + "loss": 2.7924, + "theoretical_loss": 3.7889110847829715, + "tokens_seen": 683016192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005315947843531, + "loss": 2.8288, + "theoretical_loss": 3.788874011220756, + "tokens_seen": 683081728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005215646940822, + "loss": 2.8745, + "theoretical_loss": 3.788836942211092, + "tokens_seen": 683147264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 385388, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.051872491836548, + "objective/train/theoretical_loss": 3.7887998777529845, + "objective/train/tokens_used": 703672800, + "theoretical_loss": 3.7887998777529845, + "tokens_seen": 683212800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040051153460381146, + "loss": 2.8854, + "theoretical_loss": 3.7887998777529845, + "tokens_seen": 683212800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040050150451354064, + "loss": 3.0064, + "theoretical_loss": 3.7887628178454373, + "tokens_seen": 683278336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004914744232698, + "loss": 2.973, + "theoretical_loss": 3.7887257624874557, + "tokens_seen": 683343872 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400481444332999, + "loss": 2.9739, + "theoretical_loss": 3.7886887116780446, + "tokens_seen": 683409408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004714142427282, + "loss": 2.628, + "theoretical_loss": 3.78865166541621, + "tokens_seen": 683474944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040046138415245736, + "loss": 2.7112, + "theoretical_loss": 3.7886146237009575, + "tokens_seen": 683540480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004513540621866, + "loss": 2.8486, + "theoretical_loss": 3.788577586531294, + "tokens_seen": 683606016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004413239719157, + "loss": 2.8971, + "theoretical_loss": 3.7885405539062242, + "tokens_seen": 683671552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040043129388164496, + "loss": 2.8651, + "theoretical_loss": 3.7885035258247566, + "tokens_seen": 683737088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004212637913741, + "loss": 2.8794, + "theoretical_loss": 3.788466502285898, + "tokens_seen": 683802624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004112337011033, + "loss": 2.769, + "theoretical_loss": 3.7884294832886547, + "tokens_seen": 683868160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004012036108325, + "loss": 2.9099, + "theoretical_loss": 3.788392468832036, + "tokens_seen": 683933696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003911735205617, + "loss": 2.7388, + "theoretical_loss": 3.788355458915049, + "tokens_seen": 683999232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040038114343029087, + "loss": 3.0831, + "theoretical_loss": 3.7883184535367027, + "tokens_seen": 684064768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003711133400201, + "loss": 2.6536, + "theoretical_loss": 3.7882814526960047, + "tokens_seen": 684130304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040036108324974923, + "loss": 2.7046, + "theoretical_loss": 3.7882444563919657, + "tokens_seen": 684195840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040035105315947846, + "loss": 2.6015, + "theoretical_loss": 3.7882074646235933, + "tokens_seen": 684261376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003410230692076, + "loss": 2.7814, + "theoretical_loss": 3.7881704773898974, + "tokens_seen": 684326912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003309929789368, + "loss": 2.8598, + "theoretical_loss": 3.7881334946898892, + "tokens_seen": 684392448 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400320962888666, + "loss": 2.7818, + "theoretical_loss": 3.7880965165225775, + "tokens_seen": 684457984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003109327983952, + "loss": 2.5295, + "theoretical_loss": 3.7880595428869737, + "tokens_seen": 684523520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040030090270812437, + "loss": 2.7813, + "theoretical_loss": 3.7880225737820883, + "tokens_seen": 684589056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040029087261785355, + "loss": 2.7439, + "theoretical_loss": 3.787985609206932, + "tokens_seen": 684654592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040028084252758273, + "loss": 2.8324, + "theoretical_loss": 3.7879486491605174, + "tokens_seen": 684720128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040027081243731197, + "loss": 2.7086, + "theoretical_loss": 3.7879116936418553, + "tokens_seen": 684785664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 386107, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4937210083007812, + "objective/train/theoretical_loss": 3.787874742649958, + "objective/train/tokens_used": 705311200, + "theoretical_loss": 3.787874742649958, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002607823470411, + "loss": 2.6333, + "theoretical_loss": 3.787874742649958, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040025075225677033, + "loss": 2.9428, + "theoretical_loss": 3.787837796183838, + "tokens_seen": 684916736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040024072216649946, + "loss": 2.7115, + "theoretical_loss": 3.7878008542425086, + "tokens_seen": 684982272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002306920762287, + "loss": 2.7586, + "theoretical_loss": 3.7877639168249817, + "tokens_seen": 685047808 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040022066198595787, + "loss": 2.7163, + "theoretical_loss": 3.7877269839302716, + "tokens_seen": 685113344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040021063189568705, + "loss": 2.6753, + "theoretical_loss": 3.787690055557391, + "tokens_seen": 685178880 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040020060180541623, + "loss": 2.8251, + "theoretical_loss": 3.7876531317053552, + "tokens_seen": 685244416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040019057171514547, + "loss": 2.8668, + "theoretical_loss": 3.7876162123731776, + "tokens_seen": 685309952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001805416248746, + "loss": 2.9311, + "theoretical_loss": 3.787579297559872, + "tokens_seen": 685375488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040017051153460383, + "loss": 2.853, + "theoretical_loss": 3.7875423872644545, + "tokens_seen": 685441024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040016048144433296, + "loss": 2.8186, + "theoretical_loss": 3.7875054814859404, + "tokens_seen": 685506560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001504513540622, + "loss": 2.6879, + "theoretical_loss": 3.787468580223344, + "tokens_seen": 685572096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040014042126379143, + "loss": 2.7141, + "theoretical_loss": 3.787431683475682, + "tokens_seen": 685637632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040013039117352056, + "loss": 2.6609, + "theoretical_loss": 3.7873947912419705, + "tokens_seen": 685703168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001203610832498, + "loss": 2.8451, + "theoretical_loss": 3.7873579035212255, + "tokens_seen": 685768704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001103309929789, + "loss": 2.7973, + "theoretical_loss": 3.787321020312464, + "tokens_seen": 685834240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040010030090270815, + "loss": 3.0104, + "theoretical_loss": 3.7872841416147036, + "tokens_seen": 685899776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040009027081243733, + "loss": 2.9192, + "theoretical_loss": 3.7872472674269604, + "tokens_seen": 685965312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000802407221665, + "loss": 2.7897, + "theoretical_loss": 3.787210397748253, + "tokens_seen": 686030848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000702106318957, + "loss": 2.7675, + "theoretical_loss": 3.7871735325775986, + "tokens_seen": 686096384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040006018054162493, + "loss": 2.7428, + "theoretical_loss": 3.7871366719140163, + "tokens_seen": 686161920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040005015045135406, + "loss": 2.7018, + "theoretical_loss": 3.787099815756524, + "tokens_seen": 686227456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000401203610833, + "loss": 2.7264, + "theoretical_loss": 3.787062964104141, + "tokens_seen": 686292992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000300902708124, + "loss": 2.7911, + "theoretical_loss": 3.7870261169558868, + "tokens_seen": 686358528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040002006018054166, + "loss": 2.7299, + "theoretical_loss": 3.78698927431078, + "tokens_seen": 686424064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 387264, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.96657395362854, + "objective/train/theoretical_loss": 3.786952436167841, + "objective/train/tokens_used": 706949600, + "theoretical_loss": 3.786952436167841, + "tokens_seen": 686489600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040001003009027084, + "loss": 2.6869, + "theoretical_loss": 3.786952436167841, + "tokens_seen": 686489600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004, + "loss": 2.8198, + "theoretical_loss": 3.78691560252609, + "tokens_seen": 686555136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999899699097292, + "loss": 2.5833, + "theoretical_loss": 3.7868787733845473, + "tokens_seen": 686620672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999799398194584, + "loss": 2.6918, + "theoretical_loss": 3.786841948742233, + "tokens_seen": 686686208 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039996990972918756, + "loss": 2.823, + "theoretical_loss": 3.7868051285981696, + "tokens_seen": 686751744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999598796389168, + "loss": 2.8408, + "theoretical_loss": 3.7867683129513767, + "tokens_seen": 686817280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999498495486459, + "loss": 2.7538, + "theoretical_loss": 3.786731501800877, + "tokens_seen": 686882816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039993981945837516, + "loss": 2.8252, + "theoretical_loss": 3.7866946951456923, + "tokens_seen": 686948352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999297893681043, + "loss": 2.6463, + "theoretical_loss": 3.786657892984845, + "tokens_seen": 687013888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999197592778335, + "loss": 2.8026, + "theoretical_loss": 3.786621095317357, + "tokens_seen": 687079424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999097291875627, + "loss": 2.8123, + "theoretical_loss": 3.7865843021422516, + "tokens_seen": 687144960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998996990972919, + "loss": 2.5796, + "theoretical_loss": 3.7865475134585522, + "tokens_seen": 687210496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039988966900702107, + "loss": 3.0825, + "theoretical_loss": 3.7865107292652818, + "tokens_seen": 687276032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998796389167503, + "loss": 2.7263, + "theoretical_loss": 3.786473949561465, + "tokens_seen": 687341568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039986960882647943, + "loss": 2.6653, + "theoretical_loss": 3.7864371743461254, + "tokens_seen": 687407104 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039985957873620866, + "loss": 2.7328, + "theoretical_loss": 3.786400403618287, + "tokens_seen": 687472640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998495486459378, + "loss": 2.5849, + "theoretical_loss": 3.7863636373769745, + "tokens_seen": 687538176 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399839518555667, + "loss": 2.6915, + "theoretical_loss": 3.7863268756212136, + "tokens_seen": 687603712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998294884653962, + "loss": 2.8043, + "theoretical_loss": 3.7862901183500295, + "tokens_seen": 687669248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998194583751254, + "loss": 2.9892, + "theoretical_loss": 3.7862533655624473, + "tokens_seen": 687734784 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039980942828485457, + "loss": 2.7182, + "theoretical_loss": 3.786216617257493, + "tokens_seen": 687800320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039979939819458375, + "loss": 2.7629, + "theoretical_loss": 3.786179873434193, + "tokens_seen": 687865856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039978936810431293, + "loss": 2.4144, + "theoretical_loss": 3.7861431340915743, + "tokens_seen": 687931392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039977933801404217, + "loss": 2.7272, + "theoretical_loss": 3.7861063992286637, + "tokens_seen": 687996928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997693079237713, + "loss": 2.8178, + "theoretical_loss": 3.786069668844487, + "tokens_seen": 688062464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 387908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.436974287033081, + "objective/train/theoretical_loss": 3.786032942938073, + "objective/train/tokens_used": 708588000, + "theoretical_loss": 3.786032942938073, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039975927783350053, + "loss": 2.6594, + "theoretical_loss": 3.786032942938073, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039974924774322966, + "loss": 2.7045, + "theoretical_loss": 3.785996221508449, + "tokens_seen": 688193536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997392176529589, + "loss": 2.6022, + "theoretical_loss": 3.7859595045546426, + "tokens_seen": 688259072 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039972918756268807, + "loss": 2.8448, + "theoretical_loss": 3.7859227920756835, + "tokens_seen": 688324608 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039971915747241725, + "loss": 2.8839, + "theoretical_loss": 3.785886084070599, + "tokens_seen": 688390144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039970912738214643, + "loss": 2.8146, + "theoretical_loss": 3.7858493805384184, + "tokens_seen": 688455680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039969909729187567, + "loss": 2.8004, + "theoretical_loss": 3.7858126814781707, + "tokens_seen": 688521216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996890672016048, + "loss": 2.7867, + "theoretical_loss": 3.7857759868888863, + "tokens_seen": 688586752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039967903711133403, + "loss": 3.0736, + "theoretical_loss": 3.785739296769595, + "tokens_seen": 688652288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039966900702106316, + "loss": 2.7904, + "theoretical_loss": 3.7857026111193264, + "tokens_seen": 688717824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996589769307924, + "loss": 2.7473, + "theoretical_loss": 3.785665929937111, + "tokens_seen": 688783360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996489468405216, + "loss": 2.8171, + "theoretical_loss": 3.7856292532219795, + "tokens_seen": 688848896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039963891675025076, + "loss": 2.9703, + "theoretical_loss": 3.7855925809729634, + "tokens_seen": 688914432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039962888665997994, + "loss": 2.3776, + "theoretical_loss": 3.7855559131890946, + "tokens_seen": 688979968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996188565697091, + "loss": 2.8586, + "theoretical_loss": 3.7855192498694032, + "tokens_seen": 689045504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996088264794383, + "loss": 2.7614, + "theoretical_loss": 3.785482591012922, + "tokens_seen": 689111040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039959879638916754, + "loss": 2.5907, + "theoretical_loss": 3.785445936618684, + "tokens_seen": 689176576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039958876629889666, + "loss": 2.6694, + "theoretical_loss": 3.785409286685721, + "tokens_seen": 689242112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995787362086259, + "loss": 2.7639, + "theoretical_loss": 3.785372641213066, + "tokens_seen": 689307648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995687061183551, + "loss": 2.7779, + "theoretical_loss": 3.7853360001997522, + "tokens_seen": 689373184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039955867602808426, + "loss": 2.5631, + "theoretical_loss": 3.7852993636448127, + "tokens_seen": 689438720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039954864593781344, + "loss": 2.7811, + "theoretical_loss": 3.7852627315472818, + "tokens_seen": 689504256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995386158475426, + "loss": 2.7612, + "theoretical_loss": 3.785226103906194, + "tokens_seen": 689569792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995285857572718, + "loss": 2.7939, + "theoretical_loss": 3.7851894807205833, + "tokens_seen": 689635328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039951855566700104, + "loss": 2.8221, + "theoretical_loss": 3.7851528619894843, + "tokens_seen": 689700864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 389134, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.952434778213501, + "objective/train/theoretical_loss": 3.7851162477119313, + "objective/train/tokens_used": 710226400, + "theoretical_loss": 3.7851162477119313, + "tokens_seen": 689766400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039950852557673017, + "loss": 2.9159, + "theoretical_loss": 3.7851162477119313, + "tokens_seen": 689766400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994984954864594, + "loss": 2.7995, + "theoretical_loss": 3.7850796378869607, + "tokens_seen": 689831936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039948846539618853, + "loss": 2.595, + "theoretical_loss": 3.7850430325136077, + "tokens_seen": 689897472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039947843530591776, + "loss": 2.771, + "theoretical_loss": 3.7850064315909084, + "tokens_seen": 689963008 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039946840521564694, + "loss": 2.78, + "theoretical_loss": 3.7849698351178986, + "tokens_seen": 690028544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994583751253761, + "loss": 2.7472, + "theoretical_loss": 3.784933243093615, + "tokens_seen": 690094080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994483450351053, + "loss": 2.8655, + "theoretical_loss": 3.7848966555170946, + "tokens_seen": 690159616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994383149448345, + "loss": 2.6616, + "theoretical_loss": 3.7848600723873744, + "tokens_seen": 690225152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039942828485456367, + "loss": 2.6761, + "theoretical_loss": 3.7848234937034917, + "tokens_seen": 690290688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994182547642929, + "loss": 2.7662, + "theoretical_loss": 3.784786919464484, + "tokens_seen": 690356224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039940822467402203, + "loss": 2.9131, + "theoretical_loss": 3.7847503496693893, + "tokens_seen": 690421760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039939819458375127, + "loss": 2.8552, + "theoretical_loss": 3.7847137843172467, + "tokens_seen": 690487296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993881644934805, + "loss": 2.823, + "theoretical_loss": 3.784677223407094, + "tokens_seen": 690552832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039937813440320963, + "loss": 2.7278, + "theoretical_loss": 3.78464066693797, + "tokens_seen": 690618368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039936810431293886, + "loss": 3.1157, + "theoretical_loss": 3.7846041149089142, + "tokens_seen": 690683904 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399358074222668, + "loss": 2.9657, + "theoretical_loss": 3.784567567318967, + "tokens_seen": 690749440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993480441323972, + "loss": 2.7648, + "theoretical_loss": 3.7845310241671664, + "tokens_seen": 690814976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993380140421264, + "loss": 2.6964, + "theoretical_loss": 3.7844944854525537, + "tokens_seen": 690880512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993279839518556, + "loss": 2.8395, + "theoretical_loss": 3.7844579511741694, + "tokens_seen": 690946048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039931795386158477, + "loss": 2.7431, + "theoretical_loss": 3.784421421331053, + "tokens_seen": 691011584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039930792377131395, + "loss": 2.8044, + "theoretical_loss": 3.784384895922247, + "tokens_seen": 691077120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039929789368104313, + "loss": 2.6512, + "theoretical_loss": 3.7843483749467914, + "tokens_seen": 691142656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039928786359077237, + "loss": 2.7691, + "theoretical_loss": 3.7843118584037287, + "tokens_seen": 691208192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992778335005015, + "loss": 2.7728, + "theoretical_loss": 3.7842753462921004, + "tokens_seen": 691273728 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039926780341023073, + "loss": 2.6623, + "theoretical_loss": 3.784238838610949, + "tokens_seen": 691339264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 389824, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.667067050933838, + "objective/train/theoretical_loss": 3.784202335359316, + "objective/train/tokens_used": 711864800, + "theoretical_loss": 3.784202335359316, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039925777331995986, + "loss": 2.8217, + "theoretical_loss": 3.784202335359316, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992477432296891, + "loss": 2.6805, + "theoretical_loss": 3.7841658365362454, + "tokens_seen": 691470336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039923771313941827, + "loss": 2.8809, + "theoretical_loss": 3.78412934214078, + "tokens_seen": 691535872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039922768304914745, + "loss": 2.7089, + "theoretical_loss": 3.784092852171962, + "tokens_seen": 691601408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039921765295887663, + "loss": 2.7421, + "theoretical_loss": 3.784056366628837, + "tokens_seen": 691666944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039920762286860587, + "loss": 2.7233, + "theoretical_loss": 3.7840198855104474, + "tokens_seen": 691732480 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399197592778335, + "loss": 2.5454, + "theoretical_loss": 3.783983408815838, + "tokens_seen": 691798016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039918756268806423, + "loss": 2.7803, + "theoretical_loss": 3.7839469365440537, + "tokens_seen": 691863552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039917753259779336, + "loss": 2.9658, + "theoretical_loss": 3.783910468694139, + "tokens_seen": 691929088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991675025075226, + "loss": 2.8263, + "theoretical_loss": 3.7838740052651394, + "tokens_seen": 691994624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991574724172518, + "loss": 2.8775, + "theoretical_loss": 3.783837546256099, + "tokens_seen": 692060160 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039914744232698096, + "loss": 2.5907, + "theoretical_loss": 3.7838010916660654, + "tokens_seen": 692125696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039913741223671014, + "loss": 2.9517, + "theoretical_loss": 3.783764641494084, + "tokens_seen": 692191232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991273821464393, + "loss": 2.7124, + "theoretical_loss": 3.783728195739201, + "tokens_seen": 692256768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991173520561685, + "loss": 2.6897, + "theoretical_loss": 3.7836917544004627, + "tokens_seen": 692322304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039910732196589774, + "loss": 2.8147, + "theoretical_loss": 3.7836553174769163, + "tokens_seen": 692387840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039909729187562686, + "loss": 2.8122, + "theoretical_loss": 3.783618884967609, + "tokens_seen": 692453376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990872617853561, + "loss": 2.7954, + "theoretical_loss": 3.783582456871589, + "tokens_seen": 692518912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990772316950853, + "loss": 2.767, + "theoretical_loss": 3.7835460331879034, + "tokens_seen": 692584448 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039906720160481446, + "loss": 2.5768, + "theoretical_loss": 3.7835096139155997, + "tokens_seen": 692649984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039905717151454364, + "loss": 3.0041, + "theoretical_loss": 3.783473199053728, + "tokens_seen": 692715520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990471414242728, + "loss": 2.7621, + "theoretical_loss": 3.7834367886013354, + "tokens_seen": 692781056 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399037111334002, + "loss": 2.6121, + "theoretical_loss": 3.7834003825574722, + "tokens_seen": 692846592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039902708124373124, + "loss": 2.6698, + "theoretical_loss": 3.7833639809211865, + "tokens_seen": 692912128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039901705115346037, + "loss": 2.7172, + "theoretical_loss": 3.7833275836915288, + "tokens_seen": 692977664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 391100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.815354585647583, + "objective/train/theoretical_loss": 3.7832911908675486, + "objective/train/tokens_used": 713503200, + "theoretical_loss": 3.7832911908675486, + "tokens_seen": 693043200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990070210631896, + "loss": 2.8474, + "theoretical_loss": 3.7832911908675486, + "tokens_seen": 693043200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039899699097291873, + "loss": 2.557, + "theoretical_loss": 3.7832548024482957, + "tokens_seen": 693108736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039898696088264796, + "loss": 2.4426, + "theoretical_loss": 3.783218418432822, + "tokens_seen": 693174272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039897693079237714, + "loss": 3.0426, + "theoretical_loss": 3.7831820388201765, + "tokens_seen": 693239808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989669007021063, + "loss": 2.754, + "theoretical_loss": 3.783145663609411, + "tokens_seen": 693305344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989568706118355, + "loss": 2.7348, + "theoretical_loss": 3.783109292799577, + "tokens_seen": 693370880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989468405215647, + "loss": 2.7635, + "theoretical_loss": 3.7830729263897265, + "tokens_seen": 693436416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039893681043129387, + "loss": 2.5699, + "theoretical_loss": 3.783036564378911, + "tokens_seen": 693501952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989267803410231, + "loss": 2.8455, + "theoretical_loss": 3.7830002067661814, + "tokens_seen": 693567488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039891675025075223, + "loss": 2.8716, + "theoretical_loss": 3.782963853550593, + "tokens_seen": 693633024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039890672016048147, + "loss": 2.7498, + "theoretical_loss": 3.7829275047311968, + "tokens_seen": 693698560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039889669007021065, + "loss": 2.733, + "theoretical_loss": 3.782891160307046, + "tokens_seen": 693764096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039888665997993983, + "loss": 2.7969, + "theoretical_loss": 3.782854820277195, + "tokens_seen": 693829632 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398876629889669, + "loss": 2.7611, + "theoretical_loss": 3.7828184846406963, + "tokens_seen": 693895168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988665997993982, + "loss": 2.7715, + "theoretical_loss": 3.7827821533966044, + "tokens_seen": 693960704 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039885656970912737, + "loss": 2.778, + "theoretical_loss": 3.782745826543974, + "tokens_seen": 694026240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988465396188566, + "loss": 2.9197, + "theoretical_loss": 3.7827095040818586, + "tokens_seen": 694091776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039883650952858573, + "loss": 2.6453, + "theoretical_loss": 3.7826731860093146, + "tokens_seen": 694157312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039882647943831497, + "loss": 2.7944, + "theoretical_loss": 3.782636872325396, + "tokens_seen": 694222848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988164493480441, + "loss": 2.958, + "theoretical_loss": 3.782600563029159, + "tokens_seen": 694288384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039880641925777333, + "loss": 2.7868, + "theoretical_loss": 3.782564258119659, + "tokens_seen": 694353920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987963891675025, + "loss": 2.7221, + "theoretical_loss": 3.7825279575959514, + "tokens_seen": 694419456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987863590772317, + "loss": 2.7764, + "theoretical_loss": 3.782491661457094, + "tokens_seen": 694484992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987763289869609, + "loss": 2.6213, + "theoretical_loss": 3.7824553697021415, + "tokens_seen": 694550528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039876629889669006, + "loss": 2.5601, + "theoretical_loss": 3.782419082330153, + "tokens_seen": 694616064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 392291, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8680379390716553, + "objective/train/theoretical_loss": 3.7823827993401844, + "objective/train/tokens_used": 715141600, + "theoretical_loss": 3.7823827993401844, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039875626880641924, + "loss": 2.7737, + "theoretical_loss": 3.7823827993401844, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039874623871614847, + "loss": 2.7944, + "theoretical_loss": 3.782346520731293, + "tokens_seen": 694747136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987362086258776, + "loss": 2.8212, + "theoretical_loss": 3.7823102465025373, + "tokens_seen": 694812672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039872617853560683, + "loss": 2.728, + "theoretical_loss": 3.7822739766529754, + "tokens_seen": 694878208 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398716148445336, + "loss": 2.824, + "theoretical_loss": 3.7822377111816654, + "tokens_seen": 694943744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987061183550652, + "loss": 2.633, + "theoretical_loss": 3.7822014500876664, + "tokens_seen": 695009280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986960882647944, + "loss": 2.7526, + "theoretical_loss": 3.7821651933700364, + "tokens_seen": 695074816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039868605817452356, + "loss": 2.6716, + "theoretical_loss": 3.7821289410278354, + "tokens_seen": 695140352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039867602808425274, + "loss": 2.8782, + "theoretical_loss": 3.7820926930601226, + "tokens_seen": 695205888 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398665997993982, + "loss": 2.7649, + "theoretical_loss": 3.7820564494659585, + "tokens_seen": 695271424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986559679037111, + "loss": 2.6169, + "theoretical_loss": 3.7820202102444025, + "tokens_seen": 695336960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039864593781344034, + "loss": 2.7008, + "theoretical_loss": 3.7819839753945153, + "tokens_seen": 695402496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986359077231695, + "loss": 2.9026, + "theoretical_loss": 3.7819477449153576, + "tokens_seen": 695468032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986258776328987, + "loss": 2.7479, + "theoretical_loss": 3.78191151880599, + "tokens_seen": 695533568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039861584754262794, + "loss": 2.8503, + "theoretical_loss": 3.7818752970654748, + "tokens_seen": 695599104 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039860581745235706, + "loss": 2.7765, + "theoretical_loss": 3.7818390796928725, + "tokens_seen": 695664640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985957873620863, + "loss": 2.7862, + "theoretical_loss": 3.781802866687246, + "tokens_seen": 695730176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985857572718155, + "loss": 2.9017, + "theoretical_loss": 3.7817666580476565, + "tokens_seen": 695795712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039857572718154466, + "loss": 2.6179, + "theoretical_loss": 3.781730453773167, + "tokens_seen": 695861248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039856569709127384, + "loss": 2.6699, + "theoretical_loss": 3.78169425386284, + "tokens_seen": 695926784 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398555667001003, + "loss": 2.706, + "theoretical_loss": 3.781658058315738, + "tokens_seen": 695992320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985456369107322, + "loss": 2.9331, + "theoretical_loss": 3.7816218671309256, + "tokens_seen": 696057856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039853560682046144, + "loss": 2.619, + "theoretical_loss": 3.781585680307466, + "tokens_seen": 696123392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039852557673019057, + "loss": 2.5891, + "theoretical_loss": 3.7815494978444226, + "tokens_seen": 696188928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985155466399198, + "loss": 2.785, + "theoretical_loss": 3.78151331974086, + "tokens_seen": 696254464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 392983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6465673446655273, + "objective/train/theoretical_loss": 3.781477145995842, + "objective/train/tokens_used": 716780000, + "theoretical_loss": 3.781477145995842, + "tokens_seen": 696320000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039850551654964893, + "loss": 2.5574, + "theoretical_loss": 3.781477145995842, + "tokens_seen": 696320000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039849548645937816, + "loss": 2.7248, + "theoretical_loss": 3.781440976608434, + "tokens_seen": 696385536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039848545636910734, + "loss": 2.9657, + "theoretical_loss": 3.7814048115777013, + "tokens_seen": 696451072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984754262788365, + "loss": 2.9466, + "theoretical_loss": 3.7813686509027082, + "tokens_seen": 696516608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984653961885657, + "loss": 2.8556, + "theoretical_loss": 3.7813324945825215, + "tokens_seen": 696582144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984553660982949, + "loss": 2.7166, + "theoretical_loss": 3.7812963426162067, + "tokens_seen": 696647680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039844533600802407, + "loss": 2.8797, + "theoretical_loss": 3.7812601950028295, + "tokens_seen": 696713216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984353059177533, + "loss": 2.6726, + "theoretical_loss": 3.7812240517414573, + "tokens_seen": 696778752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039842527582748243, + "loss": 2.865, + "theoretical_loss": 3.7811879128311556, + "tokens_seen": 696844288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039841524573721167, + "loss": 2.7871, + "theoretical_loss": 3.781151778270993, + "tokens_seen": 696909824 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039840521564694085, + "loss": 2.7724, + "theoretical_loss": 3.7811156480600356, + "tokens_seen": 696975360 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039839518555667003, + "loss": 2.6619, + "theoretical_loss": 3.781079522197352, + "tokens_seen": 697040896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983851554663992, + "loss": 2.6867, + "theoretical_loss": 3.7810434006820097, + "tokens_seen": 697106432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983751253761284, + "loss": 2.7643, + "theoretical_loss": 3.7810072835130764, + "tokens_seen": 697171968 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039836509528585757, + "loss": 2.6857, + "theoretical_loss": 3.7809711706896216, + "tokens_seen": 697237504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983550651955868, + "loss": 2.7306, + "theoretical_loss": 3.7809350622107134, + "tokens_seen": 697303040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039834503510531593, + "loss": 2.6711, + "theoretical_loss": 3.7808989580754213, + "tokens_seen": 697368576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039833500501504517, + "loss": 2.8561, + "theoretical_loss": 3.780862858282814, + "tokens_seen": 697434112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983249749247743, + "loss": 2.7013, + "theoretical_loss": 3.780826762831962, + "tokens_seen": 697499648 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039831494483450353, + "loss": 2.7573, + "theoretical_loss": 3.7807906717219346, + "tokens_seen": 697565184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983049147442327, + "loss": 2.8347, + "theoretical_loss": 3.7807545849518025, + "tokens_seen": 697630720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982948846539619, + "loss": 2.6078, + "theoretical_loss": 3.780718502520636, + "tokens_seen": 697696256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982848545636911, + "loss": 2.6766, + "theoretical_loss": 3.780682424427506, + "tokens_seen": 697761792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039827482447342026, + "loss": 2.6125, + "theoretical_loss": 3.780646350671484, + "tokens_seen": 697827328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039826479438314944, + "loss": 2.8044, + "theoretical_loss": 3.78061028125164, + "tokens_seen": 697892864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 394024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.852461338043213, + "objective/train/theoretical_loss": 3.7805742161670466, + "objective/train/tokens_used": 718418400, + "theoretical_loss": 3.7805742161670466, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039825476429287867, + "loss": 2.8367, + "theoretical_loss": 3.7805742161670466, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982447342026078, + "loss": 2.7577, + "theoretical_loss": 3.7805381554167763, + "tokens_seen": 698023936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039823470411233703, + "loss": 2.5646, + "theoretical_loss": 3.7805020989999, + "tokens_seen": 698089472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982246740220662, + "loss": 2.7665, + "theoretical_loss": 3.7804660469154907, + "tokens_seen": 698155008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982146439317954, + "loss": 2.8406, + "theoretical_loss": 3.7804299991626222, + "tokens_seen": 698220544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982046138415246, + "loss": 2.6331, + "theoretical_loss": 3.780393955740367, + "tokens_seen": 698286080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039819458375125376, + "loss": 2.8337, + "theoretical_loss": 3.7803579166477976, + "tokens_seen": 698351616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039818455366098294, + "loss": 2.6492, + "theoretical_loss": 3.7803218818839883, + "tokens_seen": 698417152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981745235707122, + "loss": 2.7657, + "theoretical_loss": 3.7802858514480135, + "tokens_seen": 698482688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981644934804413, + "loss": 2.8308, + "theoretical_loss": 3.780249825338947, + "tokens_seen": 698548224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039815446339017054, + "loss": 2.6012, + "theoretical_loss": 3.7802138035558634, + "tokens_seen": 698613760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039814443329989966, + "loss": 2.8382, + "theoretical_loss": 3.7801777860978367, + "tokens_seen": 698679296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981344032096289, + "loss": 2.8067, + "theoretical_loss": 3.780141772963944, + "tokens_seen": 698744832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981243731193581, + "loss": 2.8721, + "theoretical_loss": 3.7801057641532587, + "tokens_seen": 698810368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039811434302908726, + "loss": 2.538, + "theoretical_loss": 3.7800697596648565, + "tokens_seen": 698875904 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039810431293881644, + "loss": 2.9022, + "theoretical_loss": 3.7800337594978153, + "tokens_seen": 698941440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980942828485457, + "loss": 2.7558, + "theoretical_loss": 3.7799977636512088, + "tokens_seen": 699006976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980842527582748, + "loss": 2.975, + "theoretical_loss": 3.779961772124115, + "tokens_seen": 699072512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039807422266800404, + "loss": 2.7963, + "theoretical_loss": 3.7799257849156107, + "tokens_seen": 699138048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039806419257773317, + "loss": 2.838, + "theoretical_loss": 3.7798898020247726, + "tokens_seen": 699203584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980541624874624, + "loss": 2.5871, + "theoretical_loss": 3.779853823450678, + "tokens_seen": 699269120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980441323971916, + "loss": 2.8418, + "theoretical_loss": 3.779817849192405, + "tokens_seen": 699334656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039803410230692077, + "loss": 2.5972, + "theoretical_loss": 3.7797818792490303, + "tokens_seen": 699400192 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039802407221664995, + "loss": 2.8504, + "theoretical_loss": 3.7797459136196334, + "tokens_seen": 699465728 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039801404212637913, + "loss": 2.5989, + "theoretical_loss": 3.7797099523032927, + "tokens_seen": 699531264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 394602, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6187872886657715, + "objective/train/theoretical_loss": 3.7796739952990865, + "objective/train/tokens_used": 720056800, + "theoretical_loss": 3.7796739952990865, + "tokens_seen": 699596800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980040120361083, + "loss": 2.5868, + "theoretical_loss": 3.7796739952990865, + "tokens_seen": 699596800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039799398194583754, + "loss": 2.6251, + "theoretical_loss": 3.779638042606094, + "tokens_seen": 699662336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039798395185556667, + "loss": 2.7199, + "theoretical_loss": 3.779602094223394, + "tokens_seen": 699727872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979739217652959, + "loss": 2.6618, + "theoretical_loss": 3.7795661501500675, + "tokens_seen": 699793408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039796389167502503, + "loss": 2.7668, + "theoretical_loss": 3.779530210385193, + "tokens_seen": 699858944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039795386158475427, + "loss": 2.4999, + "theoretical_loss": 3.7794942749278517, + "tokens_seen": 699924480 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039794383149448345, + "loss": 2.9616, + "theoretical_loss": 3.7794583437771236, + "tokens_seen": 699990016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039793380140421263, + "loss": 2.9088, + "theoretical_loss": 3.779422416932089, + "tokens_seen": 700055552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979237713139418, + "loss": 2.8126, + "theoretical_loss": 3.77938649439183, + "tokens_seen": 700121088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039791374122367105, + "loss": 2.6539, + "theoretical_loss": 3.779350576155427, + "tokens_seen": 700186624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979037111334002, + "loss": 2.8109, + "theoretical_loss": 3.779314662221962, + "tokens_seen": 700252160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978936810431294, + "loss": 2.6834, + "theoretical_loss": 3.7792787525905167, + "tokens_seen": 700317696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978836509528586, + "loss": 2.8797, + "theoretical_loss": 3.779242847260173, + "tokens_seen": 700383232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039787362086258777, + "loss": 2.7285, + "theoretical_loss": 3.7792069462300146, + "tokens_seen": 700448768 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397863590772317, + "loss": 2.6344, + "theoretical_loss": 3.779171049499123, + "tokens_seen": 700514304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039785356068204613, + "loss": 2.7949, + "theoretical_loss": 3.779135157066581, + "tokens_seen": 700579840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039784353059177537, + "loss": 2.7319, + "theoretical_loss": 3.779099268931473, + "tokens_seen": 700645376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978335005015045, + "loss": 2.836, + "theoretical_loss": 3.779063385092882, + "tokens_seen": 700710912 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039782347041123373, + "loss": 2.7745, + "theoretical_loss": 3.779027505549891, + "tokens_seen": 700776448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978134403209629, + "loss": 2.8877, + "theoretical_loss": 3.778991630301586, + "tokens_seen": 700841984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978034102306921, + "loss": 2.7129, + "theoretical_loss": 3.77895575934705, + "tokens_seen": 700907520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977933801404213, + "loss": 2.9092, + "theoretical_loss": 3.7789198926853684, + "tokens_seen": 700973056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039778335005015046, + "loss": 2.7402, + "theoretical_loss": 3.7788840303156253, + "tokens_seen": 701038592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039777331995987964, + "loss": 2.8797, + "theoretical_loss": 3.7788481722369065, + "tokens_seen": 701104128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039776328986960887, + "loss": 2.8667, + "theoretical_loss": 3.7788123184482982, + "tokens_seen": 701169664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 395841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9108428955078125, + "objective/train/theoretical_loss": 3.7787764689488847, + "objective/train/tokens_used": 721695200, + "theoretical_loss": 3.7787764689488847, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397753259779338, + "loss": 2.8756, + "theoretical_loss": 3.7787764689488847, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039774322968906723, + "loss": 2.907, + "theoretical_loss": 3.778740623737754, + "tokens_seen": 701300736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977331995987964, + "loss": 2.5534, + "theoretical_loss": 3.778704782813991, + "tokens_seen": 701366272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977231695085256, + "loss": 2.6829, + "theoretical_loss": 3.7786689461766825, + "tokens_seen": 701431808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977131394182548, + "loss": 2.5951, + "theoretical_loss": 3.778633113824916, + "tokens_seen": 701497344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039770310932798396, + "loss": 2.9051, + "theoretical_loss": 3.778597285757778, + "tokens_seen": 701562880 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039769307923771314, + "loss": 2.7553, + "theoretical_loss": 3.7785614619743573, + "tokens_seen": 701628416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976830491474424, + "loss": 2.5643, + "theoretical_loss": 3.77852564247374, + "tokens_seen": 701693952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976730190571715, + "loss": 2.9458, + "theoretical_loss": 3.778489827255015, + "tokens_seen": 701759488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039766298896690074, + "loss": 2.8461, + "theoretical_loss": 3.778454016317271, + "tokens_seen": 701825024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039765295887662987, + "loss": 2.7984, + "theoretical_loss": 3.7784182096595957, + "tokens_seen": 701890560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976429287863591, + "loss": 2.8552, + "theoretical_loss": 3.7783824072810788, + "tokens_seen": 701956096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976328986960883, + "loss": 2.9651, + "theoretical_loss": 3.7783466091808093, + "tokens_seen": 702021632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039762286860581746, + "loss": 2.8818, + "theoretical_loss": 3.7783108153578766, + "tokens_seen": 702087168 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039761283851554664, + "loss": 2.8327, + "theoretical_loss": 3.7782750258113698, + "tokens_seen": 702152704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976028084252759, + "loss": 2.5367, + "theoretical_loss": 3.7782392405403797, + "tokens_seen": 702218240 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397592778335005, + "loss": 2.8229, + "theoretical_loss": 3.7782034595439966, + "tokens_seen": 702283776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039758274824473424, + "loss": 2.5208, + "theoretical_loss": 3.778167682821311, + "tokens_seen": 702349312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039757271815446337, + "loss": 2.695, + "theoretical_loss": 3.778131910371413, + "tokens_seen": 702414848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975626880641926, + "loss": 2.6583, + "theoretical_loss": 3.7780961421933945, + "tokens_seen": 702480384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975526579739218, + "loss": 2.774, + "theoretical_loss": 3.7780603782863467, + "tokens_seen": 702545920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039754262788365097, + "loss": 3.0157, + "theoretical_loss": 3.778024618649361, + "tokens_seen": 702611456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039753259779338015, + "loss": 2.8291, + "theoretical_loss": 3.7779888632815295, + "tokens_seen": 702676992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039752256770310933, + "loss": 2.859, + "theoretical_loss": 3.7779531121819447, + "tokens_seen": 702742528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975125376128385, + "loss": 2.8686, + "theoretical_loss": 3.7779173653496994, + "tokens_seen": 702808064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 397024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.391489028930664, + "objective/train/theoretical_loss": 3.7778816227838856, + "objective/train/tokens_used": 723333600, + "theoretical_loss": 3.7778816227838856, + "tokens_seen": 702873600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039750250752256774, + "loss": 2.806, + "theoretical_loss": 3.7778816227838856, + "tokens_seen": 702873600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039749247743229687, + "loss": 2.8732, + "theoretical_loss": 3.7778458844835963, + "tokens_seen": 702939136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974824473420261, + "loss": 2.7415, + "theoretical_loss": 3.777810150447926, + "tokens_seen": 703004672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039747241725175523, + "loss": 2.728, + "theoretical_loss": 3.777774420675967, + "tokens_seen": 703070208 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039746238716148447, + "loss": 2.6557, + "theoretical_loss": 3.7777386951668133, + "tokens_seen": 703135744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039745235707121365, + "loss": 2.6884, + "theoretical_loss": 3.7777029739195602, + "tokens_seen": 703201280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039744232698094283, + "loss": 2.8055, + "theoretical_loss": 3.7776672569333014, + "tokens_seen": 703266816 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397432296890672, + "loss": 2.6844, + "theoretical_loss": 3.7776315442071313, + "tokens_seen": 703332352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039742226680040125, + "loss": 2.7397, + "theoretical_loss": 3.7775958357401453, + "tokens_seen": 703397888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974122367101304, + "loss": 2.8919, + "theoretical_loss": 3.7775601315314393, + "tokens_seen": 703463424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974022066198596, + "loss": 2.5523, + "theoretical_loss": 3.7775244315801078, + "tokens_seen": 703528960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039739217652958874, + "loss": 2.6017, + "theoretical_loss": 3.777488735885247, + "tokens_seen": 703594496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039738214643931797, + "loss": 2.8401, + "theoretical_loss": 3.777453044445953, + "tokens_seen": 703660032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039737211634904715, + "loss": 2.7133, + "theoretical_loss": 3.7774173572613226, + "tokens_seen": 703725568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039736208625877633, + "loss": 2.6392, + "theoretical_loss": 3.777381674330452, + "tokens_seen": 703791104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973520561685055, + "loss": 2.8618, + "theoretical_loss": 3.7773459956524382, + "tokens_seen": 703856640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973420260782347, + "loss": 2.7136, + "theoretical_loss": 3.7773103212263783, + "tokens_seen": 703922176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973319959879639, + "loss": 2.7602, + "theoretical_loss": 3.7772746510513704, + "tokens_seen": 703987712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973219658976931, + "loss": 2.7088, + "theoretical_loss": 3.7772389851265116, + "tokens_seen": 704053248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039731193580742224, + "loss": 2.6738, + "theoretical_loss": 3.7772033234509, + "tokens_seen": 704118784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973019057171515, + "loss": 2.5981, + "theoretical_loss": 3.777167666023634, + "tokens_seen": 704184320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972918756268806, + "loss": 2.9311, + "theoretical_loss": 3.777132012843812, + "tokens_seen": 704249856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039728184553660984, + "loss": 2.7248, + "theoretical_loss": 3.777096363910534, + "tokens_seen": 704315392 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397271815446339, + "loss": 2.8256, + "theoretical_loss": 3.7770607192228978, + "tokens_seen": 704380928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972617853560682, + "loss": 2.8975, + "theoretical_loss": 3.777025078780003, + "tokens_seen": 704446464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 397723, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0135793685913086, + "objective/train/theoretical_loss": 3.77698944258095, + "objective/train/tokens_used": 724972000, + "theoretical_loss": 3.77698944258095, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972517552657974, + "loss": 2.7687, + "theoretical_loss": 3.77698944258095, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972417251755266, + "loss": 2.8214, + "theoretical_loss": 3.776953810624838, + "tokens_seen": 704577536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039723169508525574, + "loss": 2.4458, + "theoretical_loss": 3.7769181829107676, + "tokens_seen": 704643072 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397221664994985, + "loss": 3.0101, + "theoretical_loss": 3.7768825594378397, + "tokens_seen": 704708608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972116349047141, + "loss": 2.8727, + "theoretical_loss": 3.7768469402051545, + "tokens_seen": 704774144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039720160481444334, + "loss": 2.7502, + "theoretical_loss": 3.7768113252118134, + "tokens_seen": 704839680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971915747241725, + "loss": 2.8048, + "theoretical_loss": 3.7767757144569174, + "tokens_seen": 704905216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971815446339017, + "loss": 2.6821, + "theoretical_loss": 3.7767401079395677, + "tokens_seen": 704970752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971715145436309, + "loss": 2.7107, + "theoretical_loss": 3.7767045056588677, + "tokens_seen": 705036288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039716148445336007, + "loss": 2.9334, + "theoretical_loss": 3.776668907613918, + "tokens_seen": 705101824 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039715145436308925, + "loss": 2.7057, + "theoretical_loss": 3.7766333138038224, + "tokens_seen": 705167360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971414242728185, + "loss": 2.6521, + "theoretical_loss": 3.776597724227682, + "tokens_seen": 705232896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039713139418254766, + "loss": 2.5163, + "theoretical_loss": 3.776562138884601, + "tokens_seen": 705298432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039712136409227684, + "loss": 2.9374, + "theoretical_loss": 3.7765265577736824, + "tokens_seen": 705363968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971113340020061, + "loss": 2.6889, + "theoretical_loss": 3.7764909808940295, + "tokens_seen": 705429504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971013039117352, + "loss": 2.7154, + "theoretical_loss": 3.776455408244746, + "tokens_seen": 705495040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039709127382146444, + "loss": 2.7799, + "theoretical_loss": 3.7764198398249365, + "tokens_seen": 705560576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039708124373119357, + "loss": 2.8133, + "theoretical_loss": 3.7763842756337054, + "tokens_seen": 705626112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970712136409228, + "loss": 2.6184, + "theoretical_loss": 3.776348715670156, + "tokens_seen": 705691648 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397061183550652, + "loss": 2.6634, + "theoretical_loss": 3.7763131599333946, + "tokens_seen": 705757184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039705115346038117, + "loss": 2.6501, + "theoretical_loss": 3.776277608422526, + "tokens_seen": 705822720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039704112337011035, + "loss": 2.7362, + "theoretical_loss": 3.7762420611366556, + "tokens_seen": 705888256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039703109327983953, + "loss": 2.9123, + "theoretical_loss": 3.776206518074889, + "tokens_seen": 705953792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970210631895687, + "loss": 2.6798, + "theoretical_loss": 3.776170979236332, + "tokens_seen": 706019328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039701103309929794, + "loss": 2.7245, + "theoretical_loss": 3.7761354446200914, + "tokens_seen": 706084864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 398840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.317671775817871, + "objective/train/theoretical_loss": 3.776099914225273, + "objective/train/tokens_used": 726610400, + "theoretical_loss": 3.776099914225273, + "tokens_seen": 706150400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039700100300902707, + "loss": 2.7278, + "theoretical_loss": 3.776099914225273, + "tokens_seen": 706150400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969909729187563, + "loss": 2.8361, + "theoretical_loss": 3.7760643880509845, + "tokens_seen": 706215936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039698094282848543, + "loss": 2.8228, + "theoretical_loss": 3.776028866096332, + "tokens_seen": 706281472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039697091273821467, + "loss": 2.8389, + "theoretical_loss": 3.7759933483604238, + "tokens_seen": 706347008 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039696088264794385, + "loss": 2.7686, + "theoretical_loss": 3.775957834842367, + "tokens_seen": 706412544 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039695085255767303, + "loss": 2.8636, + "theoretical_loss": 3.775922325541269, + "tokens_seen": 706478080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969408224674022, + "loss": 2.8, + "theoretical_loss": 3.775886820456239, + "tokens_seen": 706543616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039693079237713145, + "loss": 2.8553, + "theoretical_loss": 3.7758513195863843, + "tokens_seen": 706609152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969207622868606, + "loss": 2.7078, + "theoretical_loss": 3.7758158229308143, + "tokens_seen": 706674688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969107321965898, + "loss": 2.6724, + "theoretical_loss": 3.775780330488638, + "tokens_seen": 706740224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039690070210631894, + "loss": 2.797, + "theoretical_loss": 3.7757448422589643, + "tokens_seen": 706805760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039689067201604817, + "loss": 2.8585, + "theoretical_loss": 3.7757093582409036, + "tokens_seen": 706871296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039688064192577735, + "loss": 2.5493, + "theoretical_loss": 3.7756738784335644, + "tokens_seen": 706936832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039687061183550653, + "loss": 2.6593, + "theoretical_loss": 3.775638402836057, + "tokens_seen": 707002368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968605817452357, + "loss": 2.8549, + "theoretical_loss": 3.7756029314474926, + "tokens_seen": 707067904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968505516549649, + "loss": 2.6769, + "theoretical_loss": 3.775567464266981, + "tokens_seen": 707133440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968405215646941, + "loss": 2.8655, + "theoretical_loss": 3.7755320012936338, + "tokens_seen": 707198976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968304914744233, + "loss": 2.7365, + "theoretical_loss": 3.775496542526561, + "tokens_seen": 707264512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039682046138415244, + "loss": 3.1246, + "theoretical_loss": 3.7754610879648745, + "tokens_seen": 707330048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968104312938817, + "loss": 2.9093, + "theoretical_loss": 3.775425637607687, + "tokens_seen": 707395584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968004012036108, + "loss": 2.7906, + "theoretical_loss": 3.775390191454109, + "tokens_seen": 707461120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039679037111334004, + "loss": 2.6881, + "theoretical_loss": 3.7753547495032533, + "tokens_seen": 707526656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967803410230692, + "loss": 2.6145, + "theoretical_loss": 3.775319311754232, + "tokens_seen": 707592192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967703109327984, + "loss": 2.8557, + "theoretical_loss": 3.775283878206159, + "tokens_seen": 707657728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967602808425276, + "loss": 2.8326, + "theoretical_loss": 3.7752484488581457, + "tokens_seen": 707723264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 399432, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6968586444854736, + "objective/train/theoretical_loss": 3.7752130237093064, + "objective/train/tokens_used": 728248800, + "theoretical_loss": 3.7752130237093064, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967502507522568, + "loss": 2.6011, + "theoretical_loss": 3.7752130237093064, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039674022066198594, + "loss": 2.7763, + "theoretical_loss": 3.775177602758755, + "tokens_seen": 707854336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967301905717152, + "loss": 2.9726, + "theoretical_loss": 3.7751421860056045, + "tokens_seen": 707919872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967201604814443, + "loss": 2.7084, + "theoretical_loss": 3.775106773448969, + "tokens_seen": 707985408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039671013039117354, + "loss": 2.8755, + "theoretical_loss": 3.7750713650879635, + "tokens_seen": 708050944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967001003009027, + "loss": 2.7319, + "theoretical_loss": 3.775035960921702, + "tokens_seen": 708116480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966900702106319, + "loss": 2.816, + "theoretical_loss": 3.7750005609493003, + "tokens_seen": 708182016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966800401203611, + "loss": 2.6943, + "theoretical_loss": 3.774965165169873, + "tokens_seen": 708247552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039667001003009027, + "loss": 2.7305, + "theoretical_loss": 3.774929773582535, + "tokens_seen": 708313088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039665997993981945, + "loss": 2.8117, + "theoretical_loss": 3.7748943861864026, + "tokens_seen": 708378624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966499498495487, + "loss": 2.7627, + "theoretical_loss": 3.774859002980592, + "tokens_seen": 708444160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966399197592778, + "loss": 2.8163, + "theoretical_loss": 3.774823623964219, + "tokens_seen": 708509696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039662988966900704, + "loss": 2.7808, + "theoretical_loss": 3.7747882491364004, + "tokens_seen": 708575232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039661985957873617, + "loss": 2.7834, + "theoretical_loss": 3.774752878496253, + "tokens_seen": 708640768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966098294884654, + "loss": 2.867, + "theoretical_loss": 3.7747175120428933, + "tokens_seen": 708706304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965997993981946, + "loss": 2.8166, + "theoretical_loss": 3.774682149775439, + "tokens_seen": 708771840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039658976930792377, + "loss": 2.8267, + "theoretical_loss": 3.7746467916930078, + "tokens_seen": 708837376 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039657973921765295, + "loss": 2.5516, + "theoretical_loss": 3.774611437794718, + "tokens_seen": 708902912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965697091273822, + "loss": 2.8087, + "theoretical_loss": 3.7745760880796864, + "tokens_seen": 708968448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965596790371113, + "loss": 2.8472, + "theoretical_loss": 3.7745407425470328, + "tokens_seen": 709033984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039654964894684055, + "loss": 2.6325, + "theoretical_loss": 3.774505401195875, + "tokens_seen": 709099520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965396188565697, + "loss": 2.6698, + "theoretical_loss": 3.774470064025332, + "tokens_seen": 709165056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965295887662989, + "loss": 2.4007, + "theoretical_loss": 3.774434731034524, + "tokens_seen": 709230592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965195586760281, + "loss": 2.8729, + "theoretical_loss": 3.7743994022225684, + "tokens_seen": 709296128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039650952858575727, + "loss": 2.6295, + "theoretical_loss": 3.774364077588587, + "tokens_seen": 709361664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 400468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9617061614990234, + "objective/train/theoretical_loss": 3.7743287571316984, + "objective/train/tokens_used": 729887200, + "theoretical_loss": 3.7743287571316984, + "tokens_seen": 709427200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039649949849548645, + "loss": 2.8305, + "theoretical_loss": 3.7743287571316984, + "tokens_seen": 709427200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039648946840521563, + "loss": 2.9433, + "theoretical_loss": 3.774293440851024, + "tokens_seen": 709492736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964794383149448, + "loss": 2.7475, + "theoretical_loss": 3.774258128745683, + "tokens_seen": 709558272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039646940822467405, + "loss": 2.7497, + "theoretical_loss": 3.7742228208147974, + "tokens_seen": 709623808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964593781344032, + "loss": 2.5815, + "theoretical_loss": 3.7741875170574875, + "tokens_seen": 709689344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964493480441324, + "loss": 2.7236, + "theoretical_loss": 3.774152217472875, + "tokens_seen": 709754880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964393179538616, + "loss": 2.6442, + "theoretical_loss": 3.774116922060081, + "tokens_seen": 709820416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964292878635908, + "loss": 2.8988, + "theoretical_loss": 3.7740816308182286, + "tokens_seen": 709885952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039641925777331996, + "loss": 2.9272, + "theoretical_loss": 3.7740463437464387, + "tokens_seen": 709951488 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039640922768304914, + "loss": 2.836, + "theoretical_loss": 3.7740110608438338, + "tokens_seen": 710017024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963991975927783, + "loss": 2.9205, + "theoretical_loss": 3.7739757821095363, + "tokens_seen": 710082560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039638916750250755, + "loss": 2.7784, + "theoretical_loss": 3.7739405075426706, + "tokens_seen": 710148096 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039637913741223673, + "loss": 2.7215, + "theoretical_loss": 3.7739052371423583, + "tokens_seen": 710213632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963691073219659, + "loss": 2.76, + "theoretical_loss": 3.7738699709077235, + "tokens_seen": 710279168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963590772316951, + "loss": 2.7615, + "theoretical_loss": 3.77383470883789, + "tokens_seen": 710344704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963490471414243, + "loss": 2.6549, + "theoretical_loss": 3.773799450931982, + "tokens_seen": 710410240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963390170511535, + "loss": 2.7139, + "theoretical_loss": 3.7737641971891227, + "tokens_seen": 710475776 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039632898696088264, + "loss": 2.8427, + "theoretical_loss": 3.7737289476084372, + "tokens_seen": 710541312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963189568706119, + "loss": 2.8954, + "theoretical_loss": 3.773693702189051, + "tokens_seen": 710606848 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396308926780341, + "loss": 2.6769, + "theoretical_loss": 3.773658460930088, + "tokens_seen": 710672384 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039629889669007024, + "loss": 2.7509, + "theoretical_loss": 3.7736232238306746, + "tokens_seen": 710737920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962888665997994, + "loss": 2.7514, + "theoretical_loss": 3.7735879908899355, + "tokens_seen": 710803456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962788365095286, + "loss": 2.7883, + "theoretical_loss": 3.7735527621069966, + "tokens_seen": 710868992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962688064192578, + "loss": 2.935, + "theoretical_loss": 3.773517537480984, + "tokens_seen": 710934528 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396258776328987, + "loss": 2.6263, + "theoretical_loss": 3.773482317011025, + "tokens_seen": 711000064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 401088, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.054889440536499, + "objective/train/theoretical_loss": 3.773447100696245, + "objective/train/tokens_used": 731525600, + "theoretical_loss": 3.773447100696245, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039624874623871614, + "loss": 2.9868, + "theoretical_loss": 3.773447100696245, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962387161484454, + "loss": 2.7579, + "theoretical_loss": 3.7734118885357715, + "tokens_seen": 711131136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962286860581745, + "loss": 2.8857, + "theoretical_loss": 3.7733766805287314, + "tokens_seen": 711196672 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039621865596790374, + "loss": 2.9905, + "theoretical_loss": 3.7733414766742523, + "tokens_seen": 711262208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962086258776329, + "loss": 2.584, + "theoretical_loss": 3.7733062769714616, + "tokens_seen": 711327744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961985957873621, + "loss": 2.625, + "theoretical_loss": 3.7732710814194883, + "tokens_seen": 711393280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961885656970913, + "loss": 2.8167, + "theoretical_loss": 3.7732358900174585, + "tokens_seen": 711458816 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039617853560682047, + "loss": 2.9579, + "theoretical_loss": 3.7732007027645027, + "tokens_seen": 711524352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039616850551654965, + "loss": 2.876, + "theoretical_loss": 3.7731655196597496, + "tokens_seen": 711589888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961584754262789, + "loss": 2.757, + "theoretical_loss": 3.7731303407023264, + "tokens_seen": 711655424 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396148445336008, + "loss": 2.8455, + "theoretical_loss": 3.773095165891364, + "tokens_seen": 711720960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039613841524573724, + "loss": 2.6882, + "theoretical_loss": 3.7730599952259913, + "tokens_seen": 711786496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039612838515546637, + "loss": 2.7801, + "theoretical_loss": 3.7730248287053376, + "tokens_seen": 711852032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961183550651956, + "loss": 2.8345, + "theoretical_loss": 3.772989666328534, + "tokens_seen": 711917568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961083249749248, + "loss": 2.8088, + "theoretical_loss": 3.7729545080947102, + "tokens_seen": 711983104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039609829488465397, + "loss": 2.4923, + "theoretical_loss": 3.772919354002997, + "tokens_seen": 712048640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039608826479438315, + "loss": 2.6372, + "theoretical_loss": 3.772884204052525, + "tokens_seen": 712114176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960782347041124, + "loss": 2.7558, + "theoretical_loss": 3.7728490582424254, + "tokens_seen": 712179712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960682046138415, + "loss": 2.7638, + "theoretical_loss": 3.7728139165718297, + "tokens_seen": 712245248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039605817452357075, + "loss": 2.875, + "theoretical_loss": 3.7727787790398697, + "tokens_seen": 712310784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960481444332999, + "loss": 2.9478, + "theoretical_loss": 3.7727436456456767, + "tokens_seen": 712376320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960381143430291, + "loss": 2.7149, + "theoretical_loss": 3.772708516388383, + "tokens_seen": 712441856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960280842527583, + "loss": 2.8476, + "theoretical_loss": 3.772673391267121, + "tokens_seen": 712507392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039601805416248747, + "loss": 2.7814, + "theoretical_loss": 3.772638270281024, + "tokens_seen": 712572928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039600802407221665, + "loss": 2.7704, + "theoretical_loss": 3.7726031534292246, + "tokens_seen": 712638464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 402378, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6918859481811523, + "objective/train/theoretical_loss": 3.7725680407108557, + "objective/train/tokens_used": 733164000, + "theoretical_loss": 3.7725680407108557, + "tokens_seen": 712704000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039599799398194583, + "loss": 2.7392, + "theoretical_loss": 3.7725680407108557, + "tokens_seen": 712704000 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395987963891675, + "loss": 2.6279, + "theoretical_loss": 3.7725329321250505, + "tokens_seen": 712769536 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039597793380140425, + "loss": 2.7209, + "theoretical_loss": 3.7724978276709433, + "tokens_seen": 712835072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959679037111334, + "loss": 2.7042, + "theoretical_loss": 3.772462727347668, + "tokens_seen": 712900608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959578736208626, + "loss": 2.6791, + "theoretical_loss": 3.7724276311543585, + "tokens_seen": 712966144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959478435305918, + "loss": 2.8125, + "theoretical_loss": 3.7723925390901503, + "tokens_seen": 713031680 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395937813440321, + "loss": 2.7226, + "theoretical_loss": 3.7723574511541766, + "tokens_seen": 713097216 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039592778335005016, + "loss": 2.6501, + "theoretical_loss": 3.7723223673455735, + "tokens_seen": 713162752 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039591775325977934, + "loss": 2.8047, + "theoretical_loss": 3.772287287663476, + "tokens_seen": 713228288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959077231695085, + "loss": 2.7207, + "theoretical_loss": 3.77225221210702, + "tokens_seen": 713293824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039589769307923775, + "loss": 2.7146, + "theoretical_loss": 3.7722171406753406, + "tokens_seen": 713359360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958876629889669, + "loss": 2.713, + "theoretical_loss": 3.772182073367574, + "tokens_seen": 713424896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958776328986961, + "loss": 2.8498, + "theoretical_loss": 3.7721470101828576, + "tokens_seen": 713490432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039586760280842524, + "loss": 2.3899, + "theoretical_loss": 3.7721119511203263, + "tokens_seen": 713555968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958575727181545, + "loss": 2.5299, + "theoretical_loss": 3.7720768961791182, + "tokens_seen": 713621504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039584754262788366, + "loss": 2.6941, + "theoretical_loss": 3.7720418453583697, + "tokens_seen": 713687040 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039583751253761284, + "loss": 2.6472, + "theoretical_loss": 3.7720067986572188, + "tokens_seen": 713752576 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395827482447342, + "loss": 2.7839, + "theoretical_loss": 3.7719717560748025, + "tokens_seen": 713818112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958174523570712, + "loss": 2.7576, + "theoretical_loss": 3.771936717610259, + "tokens_seen": 713883648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958074222668004, + "loss": 2.722, + "theoretical_loss": 3.7719016832627266, + "tokens_seen": 713949184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957973921765296, + "loss": 2.44, + "theoretical_loss": 3.7718666530313434, + "tokens_seen": 714014720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039578736208625875, + "loss": 2.7879, + "theoretical_loss": 3.7718316269152483, + "tokens_seen": 714080256 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395777331995988, + "loss": 3.0367, + "theoretical_loss": 3.77179660491358, + "tokens_seen": 714145792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039576730190571716, + "loss": 2.8812, + "theoretical_loss": 3.7717615870254777, + "tokens_seen": 714211328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039575727181544634, + "loss": 2.9063, + "theoretical_loss": 3.7717265732500804, + "tokens_seen": 714276864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 402960, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7978687286376953, + "objective/train/theoretical_loss": 3.771691563586529, + "objective/train/tokens_used": 734802400, + "theoretical_loss": 3.771691563586529, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957472417251755, + "loss": 2.6231, + "theoretical_loss": 3.771691563586529, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957372116349047, + "loss": 2.7291, + "theoretical_loss": 3.7716565580339623, + "tokens_seen": 714407936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957271815446339, + "loss": 2.7152, + "theoretical_loss": 3.771621556591522, + "tokens_seen": 714473472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957171514543631, + "loss": 2.7035, + "theoretical_loss": 3.7715865592583464, + "tokens_seen": 714539008 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039570712136409225, + "loss": 2.6655, + "theoretical_loss": 3.771551566033578, + "tokens_seen": 714604544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956970912738215, + "loss": 2.6651, + "theoretical_loss": 3.771516576916357, + "tokens_seen": 714670080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956870611835506, + "loss": 2.6674, + "theoretical_loss": 3.7714815919058244, + "tokens_seen": 714735616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039567703109327985, + "loss": 2.7265, + "theoretical_loss": 3.771446611001123, + "tokens_seen": 714801152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039566700100300903, + "loss": 2.6466, + "theoretical_loss": 3.7714116342013932, + "tokens_seen": 714866688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956569709127382, + "loss": 2.8245, + "theoretical_loss": 3.7713766615057778, + "tokens_seen": 714932224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956469408224674, + "loss": 2.7489, + "theoretical_loss": 3.7713416929134187, + "tokens_seen": 714997760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039563691073219657, + "loss": 2.6609, + "theoretical_loss": 3.7713067284234585, + "tokens_seen": 715063296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956268806419258, + "loss": 2.6133, + "theoretical_loss": 3.7712717680350405, + "tokens_seen": 715128832 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395616850551655, + "loss": 2.7701, + "theoretical_loss": 3.7712368117473067, + "tokens_seen": 715194368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039560682046138417, + "loss": 2.5693, + "theoretical_loss": 3.771201859559401, + "tokens_seen": 715259904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039559679037111335, + "loss": 2.9091, + "theoretical_loss": 3.771166911470468, + "tokens_seen": 715325440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955867602808426, + "loss": 2.6383, + "theoretical_loss": 3.7711319674796497, + "tokens_seen": 715390976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955767301905717, + "loss": 2.8301, + "theoretical_loss": 3.771097027586091, + "tokens_seen": 715456512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039556670010030095, + "loss": 2.8646, + "theoretical_loss": 3.7710620917889366, + "tokens_seen": 715522048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955566700100301, + "loss": 2.664, + "theoretical_loss": 3.77102716008733, + "tokens_seen": 715587584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955466399197593, + "loss": 2.5999, + "theoretical_loss": 3.7709922324804177, + "tokens_seen": 715653120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955366098294885, + "loss": 2.9666, + "theoretical_loss": 3.7709573089673443, + "tokens_seen": 715718656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039552657973921767, + "loss": 2.6747, + "theoretical_loss": 3.7709223895472537, + "tokens_seen": 715784192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039551654964894685, + "loss": 2.4762, + "theoretical_loss": 3.7708874742192933, + "tokens_seen": 715849728 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039550651955867603, + "loss": 2.8351, + "theoretical_loss": 3.7708525629826077, + "tokens_seen": 715915264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 403589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7969202995300293, + "objective/train/theoretical_loss": 3.7708176558363444, + "objective/train/tokens_used": 736440800, + "theoretical_loss": 3.7708176558363444, + "tokens_seen": 715980800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954964894684052, + "loss": 2.5087, + "theoretical_loss": 3.7708176558363444, + "tokens_seen": 715980800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039548645937813445, + "loss": 2.7577, + "theoretical_loss": 3.770782752779649, + "tokens_seen": 716046336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954764292878636, + "loss": 2.8457, + "theoretical_loss": 3.7707478538116677, + "tokens_seen": 716111872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954663991975928, + "loss": 2.5495, + "theoretical_loss": 3.7707129589315485, + "tokens_seen": 716177408 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395456369107322, + "loss": 2.7968, + "theoretical_loss": 3.7706780681384373, + "tokens_seen": 716242944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954463390170512, + "loss": 2.5126, + "theoretical_loss": 3.770643181431483, + "tokens_seen": 716308480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039543630892678036, + "loss": 2.7764, + "theoretical_loss": 3.770608298809832, + "tokens_seen": 716374016 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039542627883650954, + "loss": 2.7939, + "theoretical_loss": 3.770573420272633, + "tokens_seen": 716439552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954162487462387, + "loss": 2.6007, + "theoretical_loss": 3.7705385458190337, + "tokens_seen": 716505088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039540621865596795, + "loss": 2.7519, + "theoretical_loss": 3.7705036754481824, + "tokens_seen": 716570624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953961885656971, + "loss": 2.9651, + "theoretical_loss": 3.7704688091592287, + "tokens_seen": 716636160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953861584754263, + "loss": 2.511, + "theoretical_loss": 3.7704339469513206, + "tokens_seen": 716701696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039537612838515544, + "loss": 2.6907, + "theoretical_loss": 3.770399088823608, + "tokens_seen": 716767232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953660982948847, + "loss": 2.5684, + "theoretical_loss": 3.77036423477524, + "tokens_seen": 716832768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039535606820461386, + "loss": 2.6212, + "theoretical_loss": 3.7703293848053665, + "tokens_seen": 716898304 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039534603811434304, + "loss": 2.618, + "theoretical_loss": 3.770294538913137, + "tokens_seen": 716963840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953360080240722, + "loss": 2.6721, + "theoretical_loss": 3.7702596970977025, + "tokens_seen": 717029376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953259779338014, + "loss": 2.7419, + "theoretical_loss": 3.770224859358213, + "tokens_seen": 717094912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953159478435306, + "loss": 2.7789, + "theoretical_loss": 3.770190025693819, + "tokens_seen": 717160448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953059177532598, + "loss": 2.6623, + "theoretical_loss": 3.7701551961036723, + "tokens_seen": 717225984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039529588766298895, + "loss": 2.9295, + "theoretical_loss": 3.7701203705869233, + "tokens_seen": 717291520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952858575727182, + "loss": 2.7038, + "theoretical_loss": 3.770085549142724, + "tokens_seen": 717357056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039527582748244736, + "loss": 2.759, + "theoretical_loss": 3.770050731770226, + "tokens_seen": 717422592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039526579739217654, + "loss": 2.4058, + "theoretical_loss": 3.770015918468582, + "tokens_seen": 717488128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952557673019057, + "loss": 2.7957, + "theoretical_loss": 3.7699811092369426, + "tokens_seen": 717553664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 404678, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7875523567199707, + "objective/train/theoretical_loss": 3.7699463040744616, + "objective/train/tokens_used": 738079200, + "theoretical_loss": 3.7699463040744616, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952457372116349, + "loss": 2.6614, + "theoretical_loss": 3.7699463040744616, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952357071213641, + "loss": 2.672, + "theoretical_loss": 3.7699115029802917, + "tokens_seen": 717684736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952256770310933, + "loss": 2.7551, + "theoretical_loss": 3.769876705953586, + "tokens_seen": 717750272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039521564694082245, + "loss": 2.6962, + "theoretical_loss": 3.769841912993497, + "tokens_seen": 717815808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952056168505517, + "loss": 2.7611, + "theoretical_loss": 3.769807124099179, + "tokens_seen": 717881344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951955867602808, + "loss": 2.8147, + "theoretical_loss": 3.7697723392697853, + "tokens_seen": 717946880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039518555667001005, + "loss": 2.7761, + "theoretical_loss": 3.7697375585044703, + "tokens_seen": 718012416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039517552657973923, + "loss": 2.8367, + "theoretical_loss": 3.769702781802388, + "tokens_seen": 718077952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951654964894684, + "loss": 2.6044, + "theoretical_loss": 3.769668009162693, + "tokens_seen": 718143488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951554663991976, + "loss": 2.5959, + "theoretical_loss": 3.7696332405845405, + "tokens_seen": 718209024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039514543630892677, + "loss": 2.517, + "theoretical_loss": 3.7695984760670855, + "tokens_seen": 718274560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039513540621865595, + "loss": 2.7682, + "theoretical_loss": 3.7695637156094826, + "tokens_seen": 718340096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951253761283852, + "loss": 2.4989, + "theoretical_loss": 3.769528959210888, + "tokens_seen": 718405632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951153460381143, + "loss": 2.647, + "theoretical_loss": 3.7694942068704576, + "tokens_seen": 718471168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039510531594784355, + "loss": 2.735, + "theoretical_loss": 3.7694594585873467, + "tokens_seen": 718536704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039509528585757273, + "loss": 2.6373, + "theoretical_loss": 3.7694247143607127, + "tokens_seen": 718602240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950852557673019, + "loss": 2.8542, + "theoretical_loss": 3.7693899741897106, + "tokens_seen": 718667776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950752256770311, + "loss": 2.843, + "theoretical_loss": 3.7693552380734987, + "tokens_seen": 718733312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950651955867603, + "loss": 2.7613, + "theoretical_loss": 3.7693205060112343, + "tokens_seen": 718798848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039505516549648946, + "loss": 2.7541, + "theoretical_loss": 3.769285778002073, + "tokens_seen": 718864384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950451354062187, + "loss": 2.5525, + "theoretical_loss": 3.769251054045174, + "tokens_seen": 718929920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950351053159478, + "loss": 2.7406, + "theoretical_loss": 3.769216334139694, + "tokens_seen": 718995456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039502507522567705, + "loss": 2.6661, + "theoretical_loss": 3.769181618284792, + "tokens_seen": 719060992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950150451354062, + "loss": 2.4692, + "theoretical_loss": 3.7691469064796257, + "tokens_seen": 719126528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950050150451354, + "loss": 2.5154, + "theoretical_loss": 3.769112198723354, + "tokens_seen": 719192064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 405859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0279910564422607, + "objective/train/theoretical_loss": 3.769077495015136, + "objective/train/tokens_used": 739717600, + "theoretical_loss": 3.769077495015136, + "tokens_seen": 719257600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949949849548646, + "loss": 2.6503, + "theoretical_loss": 3.769077495015136, + "tokens_seen": 719257600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949849548645938, + "loss": 2.8353, + "theoretical_loss": 3.76904279535413, + "tokens_seen": 719323136 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039497492477432296, + "loss": 2.6663, + "theoretical_loss": 3.7690080997394952, + "tokens_seen": 719388672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949648946840522, + "loss": 2.7235, + "theoretical_loss": 3.7689734081703925, + "tokens_seen": 719454208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949548645937813, + "loss": 2.5129, + "theoretical_loss": 3.7689387206459806, + "tokens_seen": 719519744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039494483450351056, + "loss": 2.8694, + "theoretical_loss": 3.7689040371654206, + "tokens_seen": 719585280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949348044132397, + "loss": 2.801, + "theoretical_loss": 3.768869357727872, + "tokens_seen": 719650816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949247743229689, + "loss": 2.8021, + "theoretical_loss": 3.7688346823324954, + "tokens_seen": 719716352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949147442326981, + "loss": 2.8085, + "theoretical_loss": 3.7688000109784516, + "tokens_seen": 719781888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949047141424273, + "loss": 2.6279, + "theoretical_loss": 3.768765343664902, + "tokens_seen": 719847424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948946840521565, + "loss": 2.6426, + "theoretical_loss": 3.7687306803910086, + "tokens_seen": 719912960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039488465396188564, + "loss": 2.7365, + "theoretical_loss": 3.7686960211559315, + "tokens_seen": 719978496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948746238716149, + "loss": 2.8506, + "theoretical_loss": 3.7686613659588337, + "tokens_seen": 720044032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039486459378134406, + "loss": 2.9218, + "theoretical_loss": 3.7686267147988772, + "tokens_seen": 720109568 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039485456369107324, + "loss": 2.7985, + "theoretical_loss": 3.7685920676752236, + "tokens_seen": 720175104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948445336008024, + "loss": 2.7395, + "theoretical_loss": 3.768557424587036, + "tokens_seen": 720240640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948345035105316, + "loss": 2.9131, + "theoretical_loss": 3.7685227855334777, + "tokens_seen": 720306176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948244734202608, + "loss": 2.7174, + "theoretical_loss": 3.7684881505137113, + "tokens_seen": 720371712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039481444332999, + "loss": 3.0072, + "theoretical_loss": 3.7684535195268998, + "tokens_seen": 720437248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039480441323971915, + "loss": 2.7996, + "theoretical_loss": 3.7684188925722073, + "tokens_seen": 720502784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947943831494484, + "loss": 2.8799, + "theoretical_loss": 3.7683842696487977, + "tokens_seen": 720568320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039478435305917756, + "loss": 2.6279, + "theoretical_loss": 3.7683496507558347, + "tokens_seen": 720633856 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039477432296890674, + "loss": 3.06, + "theoretical_loss": 3.7683150358924826, + "tokens_seen": 720699392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947642928786359, + "loss": 3.0075, + "theoretical_loss": 3.7682804250579065, + "tokens_seen": 720764928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947542627883651, + "loss": 2.9534, + "theoretical_loss": 3.768245818251271, + "tokens_seen": 720830464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 406138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.268052577972412, + "objective/train/theoretical_loss": 3.768211215471741, + "objective/train/tokens_used": 741356000, + "theoretical_loss": 3.768211215471741, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947442326980943, + "loss": 2.7847, + "theoretical_loss": 3.768211215471741, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947342026078235, + "loss": 2.6443, + "theoretical_loss": 3.768176616718482, + "tokens_seen": 720961536 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039472417251755265, + "loss": 2.5376, + "theoretical_loss": 3.7681420219906596, + "tokens_seen": 721027072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947141424272819, + "loss": 2.579, + "theoretical_loss": 3.76810743128744, + "tokens_seen": 721092608 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394704112337011, + "loss": 2.7805, + "theoretical_loss": 3.768072844607988, + "tokens_seen": 721158144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039469408224674025, + "loss": 2.6879, + "theoretical_loss": 3.768038261951472, + "tokens_seen": 721223680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039468405215646943, + "loss": 2.6076, + "theoretical_loss": 3.7680036833170565, + "tokens_seen": 721289216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946740220661986, + "loss": 2.699, + "theoretical_loss": 3.7679691087039098, + "tokens_seen": 721354752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946639919759278, + "loss": 2.7964, + "theoretical_loss": 3.7679345381111986, + "tokens_seen": 721420288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039465396188565697, + "loss": 2.5689, + "theoretical_loss": 3.76789997153809, + "tokens_seen": 721485824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039464393179538615, + "loss": 2.7629, + "theoretical_loss": 3.767865408983751, + "tokens_seen": 721551360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946339017051154, + "loss": 2.6272, + "theoretical_loss": 3.7678308504473508, + "tokens_seen": 721616896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946238716148445, + "loss": 2.581, + "theoretical_loss": 3.767796295928056, + "tokens_seen": 721682432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039461384152457375, + "loss": 2.7804, + "theoretical_loss": 3.767761745425036, + "tokens_seen": 721747968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039460381143430293, + "loss": 2.8805, + "theoretical_loss": 3.767727198937459, + "tokens_seen": 721813504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945937813440321, + "loss": 2.8968, + "theoretical_loss": 3.767692656464494, + "tokens_seen": 721879040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945837512537613, + "loss": 2.6535, + "theoretical_loss": 3.76765811800531, + "tokens_seen": 721944576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945737211634905, + "loss": 2.8241, + "theoretical_loss": 3.767623583559076, + "tokens_seen": 722010112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039456369107321966, + "loss": 2.9034, + "theoretical_loss": 3.767589053124962, + "tokens_seen": 722075648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945536609829489, + "loss": 3.0686, + "theoretical_loss": 3.767554526702137, + "tokens_seen": 722141184 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394543630892678, + "loss": 2.8172, + "theoretical_loss": 3.7675200042897723, + "tokens_seen": 722206720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039453360080240725, + "loss": 2.8621, + "theoretical_loss": 3.767485485887037, + "tokens_seen": 722272256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945235707121364, + "loss": 2.8103, + "theoretical_loss": 3.7674509714931026, + "tokens_seen": 722337792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945135406218656, + "loss": 2.893, + "theoretical_loss": 3.767416461107139, + "tokens_seen": 722403328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945035105315948, + "loss": 2.8334, + "theoretical_loss": 3.767381954728318, + "tokens_seen": 722468864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 407490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3906686305999756, + "objective/train/theoretical_loss": 3.7673474523558106, + "objective/train/tokens_used": 742994400, + "theoretical_loss": 3.7673474523558106, + "tokens_seen": 722534400 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394493480441324, + "loss": 2.661, + "theoretical_loss": 3.7673474523558106, + "tokens_seen": 722534400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039448345035105316, + "loss": 2.7209, + "theoretical_loss": 3.767312953988788, + "tokens_seen": 722599936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944734202607824, + "loss": 2.5782, + "theoretical_loss": 3.767278459626423, + "tokens_seen": 722665472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944633901705115, + "loss": 2.589, + "theoretical_loss": 3.767243969267886, + "tokens_seen": 722731008 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039445336008024076, + "loss": 2.7244, + "theoretical_loss": 3.7672094829123512, + "tokens_seen": 722796544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944433299899699, + "loss": 2.6615, + "theoretical_loss": 3.7671750005589892, + "tokens_seen": 722862080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944332998996991, + "loss": 3.0289, + "theoretical_loss": 3.767140522206974, + "tokens_seen": 722927616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944232698094283, + "loss": 2.6406, + "theoretical_loss": 3.7671060478554783, + "tokens_seen": 722993152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944132397191575, + "loss": 2.746, + "theoretical_loss": 3.7670715775036756, + "tokens_seen": 723058688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039440320962888666, + "loss": 2.9204, + "theoretical_loss": 3.7670371111507386, + "tokens_seen": 723124224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039439317953861584, + "loss": 2.5796, + "theoretical_loss": 3.7670026487958417, + "tokens_seen": 723189760 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394383149448345, + "loss": 2.6218, + "theoretical_loss": 3.766968190438159, + "tokens_seen": 723255296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039437311935807426, + "loss": 2.7064, + "theoretical_loss": 3.766933736076864, + "tokens_seen": 723320832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943630892678034, + "loss": 2.7234, + "theoretical_loss": 3.766899285711132, + "tokens_seen": 723386368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943530591775326, + "loss": 3.1493, + "theoretical_loss": 3.766864839340138, + "tokens_seen": 723451904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039434302908726175, + "loss": 2.6272, + "theoretical_loss": 3.7668303969630554, + "tokens_seen": 723517440 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394332998996991, + "loss": 2.9986, + "theoretical_loss": 3.7667959585790607, + "tokens_seen": 723582976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039432296890672017, + "loss": 2.8482, + "theoretical_loss": 3.76676152418733, + "tokens_seen": 723648512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039431293881644935, + "loss": 2.808, + "theoretical_loss": 3.7667270937870363, + "tokens_seen": 723714048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039430290872617853, + "loss": 2.7588, + "theoretical_loss": 3.7666926673773586, + "tokens_seen": 723779584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039429287863590776, + "loss": 2.6425, + "theoretical_loss": 3.7666582449574717, + "tokens_seen": 723845120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942828485456369, + "loss": 2.6043, + "theoretical_loss": 3.766623826526552, + "tokens_seen": 723910656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942728184553661, + "loss": 2.3893, + "theoretical_loss": 3.766589412083776, + "tokens_seen": 723976192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039426278836509525, + "loss": 2.8363, + "theoretical_loss": 3.766555001628322, + "tokens_seen": 724041728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942527582748245, + "loss": 2.7142, + "theoretical_loss": 3.766520595159365, + "tokens_seen": 724107264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 408057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.163313150405884, + "objective/train/theoretical_loss": 3.766486192676084, + "objective/train/tokens_used": 744632800, + "theoretical_loss": 3.766486192676084, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039424272818455367, + "loss": 2.7706, + "theoretical_loss": 3.766486192676084, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039423269809428285, + "loss": 2.7204, + "theoretical_loss": 3.766451794177656, + "tokens_seen": 724238336 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039422266800401203, + "loss": 2.8968, + "theoretical_loss": 3.7664173996632595, + "tokens_seen": 724303872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942126379137412, + "loss": 2.7789, + "theoretical_loss": 3.7663830091320722, + "tokens_seen": 724369408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942026078234704, + "loss": 2.8649, + "theoretical_loss": 3.7663486225832727, + "tokens_seen": 724434944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039419257773319963, + "loss": 3.0287, + "theoretical_loss": 3.7663142400160403, + "tokens_seen": 724500480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039418254764292876, + "loss": 2.9706, + "theoretical_loss": 3.7662798614295516, + "tokens_seen": 724566016 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394172517552658, + "loss": 2.5185, + "theoretical_loss": 3.7662454868229887, + "tokens_seen": 724631552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941624874623871, + "loss": 2.6261, + "theoretical_loss": 3.766211116195528, + "tokens_seen": 724697088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039415245737211635, + "loss": 2.7275, + "theoretical_loss": 3.766176749546352, + "tokens_seen": 724762624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941424272818456, + "loss": 2.8693, + "theoretical_loss": 3.7661423868746384, + "tokens_seen": 724828160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941323971915747, + "loss": 2.6889, + "theoretical_loss": 3.766108028179568, + "tokens_seen": 724893696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039412236710130395, + "loss": 2.7965, + "theoretical_loss": 3.7660736734603217, + "tokens_seen": 724959232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039411233701103313, + "loss": 2.7244, + "theoretical_loss": 3.7660393227160798, + "tokens_seen": 725024768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941023069207623, + "loss": 2.8347, + "theoretical_loss": 3.7660049759460223, + "tokens_seen": 725090304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940922768304915, + "loss": 2.7925, + "theoretical_loss": 3.7659706331493314, + "tokens_seen": 725155840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940822467402207, + "loss": 2.8973, + "theoretical_loss": 3.7659362943251877, + "tokens_seen": 725221376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039407221664994986, + "loss": 2.4894, + "theoretical_loss": 3.765901959472773, + "tokens_seen": 725286912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940621865596791, + "loss": 2.7852, + "theoretical_loss": 3.7658676285912693, + "tokens_seen": 725352448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940521564694082, + "loss": 2.8938, + "theoretical_loss": 3.765833301679858, + "tokens_seen": 725417984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039404212637913745, + "loss": 2.7822, + "theoretical_loss": 3.765798978737722, + "tokens_seen": 725483520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940320962888666, + "loss": 2.7152, + "theoretical_loss": 3.765764659764044, + "tokens_seen": 725549056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940220661985958, + "loss": 2.7457, + "theoretical_loss": 3.7657303447580066, + "tokens_seen": 725614592 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394012036108325, + "loss": 2.7628, + "theoretical_loss": 3.765696033718792, + "tokens_seen": 725680128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940020060180542, + "loss": 2.8026, + "theoretical_loss": 3.7656617266455843, + "tokens_seen": 725745664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 409133, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.649945020675659, + "objective/train/theoretical_loss": 3.765627423537567, + "objective/train/tokens_used": 746271200, + "theoretical_loss": 3.765627423537567, + "tokens_seen": 725811200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039399197592778336, + "loss": 2.6373, + "theoretical_loss": 3.765627423537567, + "tokens_seen": 725811200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939819458375126, + "loss": 2.9122, + "theoretical_loss": 3.765593124393923, + "tokens_seen": 725876736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939719157472417, + "loss": 2.8708, + "theoretical_loss": 3.7655588292138376, + "tokens_seen": 725942272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039396188565697096, + "loss": 2.7742, + "theoretical_loss": 3.7655245379964937, + "tokens_seen": 726007808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939518555667001, + "loss": 2.6562, + "theoretical_loss": 3.765490250741077, + "tokens_seen": 726073344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939418254764293, + "loss": 2.6231, + "theoretical_loss": 3.765455967446771, + "tokens_seen": 726138880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939317953861585, + "loss": 2.758, + "theoretical_loss": 3.7654216881127613, + "tokens_seen": 726204416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939217652958877, + "loss": 2.7845, + "theoretical_loss": 3.7653874127382334, + "tokens_seen": 726269952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039391173520561686, + "loss": 2.8462, + "theoretical_loss": 3.765353141322372, + "tokens_seen": 726335488 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039390170511534604, + "loss": 2.6643, + "theoretical_loss": 3.7653188738643637, + "tokens_seen": 726401024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938916750250752, + "loss": 2.9446, + "theoretical_loss": 3.765284610363393, + "tokens_seen": 726466560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039388164493480446, + "loss": 2.9237, + "theoretical_loss": 3.765250350818647, + "tokens_seen": 726532096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938716148445336, + "loss": 2.5889, + "theoretical_loss": 3.7652160952293126, + "tokens_seen": 726597632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938615847542628, + "loss": 2.7943, + "theoretical_loss": 3.765181843594575, + "tokens_seen": 726663168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039385155466399195, + "loss": 2.8595, + "theoretical_loss": 3.7651475959136222, + "tokens_seen": 726728704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938415245737212, + "loss": 2.7223, + "theoretical_loss": 3.765113352185641, + "tokens_seen": 726794240 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039383149448345037, + "loss": 2.4638, + "theoretical_loss": 3.7650791124098193, + "tokens_seen": 726859776 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039382146439317955, + "loss": 2.8602, + "theoretical_loss": 3.7650448765853435, + "tokens_seen": 726925312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039381143430290873, + "loss": 2.732, + "theoretical_loss": 3.765010644711402, + "tokens_seen": 726990848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039380140421263796, + "loss": 2.8741, + "theoretical_loss": 3.7649764167871824, + "tokens_seen": 727056384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937913741223671, + "loss": 2.7782, + "theoretical_loss": 3.7649421928118745, + "tokens_seen": 727121920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937813440320963, + "loss": 2.6986, + "theoretical_loss": 3.7649079727846653, + "tokens_seen": 727187456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039377131394182545, + "loss": 3.0349, + "theoretical_loss": 3.764873756704745, + "tokens_seen": 727252992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937612838515547, + "loss": 2.8043, + "theoretical_loss": 3.764839544571301, + "tokens_seen": 727318528 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039375125376128387, + "loss": 2.7541, + "theoretical_loss": 3.7648053363835237, + "tokens_seen": 727384064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 409693, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.393568992614746, + "objective/train/theoretical_loss": 3.764771132140602, + "objective/train/tokens_used": 747909600, + "theoretical_loss": 3.764771132140602, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039374122367101305, + "loss": 2.8103, + "theoretical_loss": 3.764771132140602, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039373119358074223, + "loss": 2.768, + "theoretical_loss": 3.7647369318417265, + "tokens_seen": 727515136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937211634904714, + "loss": 2.853, + "theoretical_loss": 3.764702735486086, + "tokens_seen": 727580672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937111334002006, + "loss": 2.7399, + "theoretical_loss": 3.764668543072872, + "tokens_seen": 727646208 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039370110330992983, + "loss": 2.8607, + "theoretical_loss": 3.764634354601274, + "tokens_seen": 727711744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039369107321965896, + "loss": 2.9141, + "theoretical_loss": 3.7646001700704836, + "tokens_seen": 727777280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936810431293882, + "loss": 2.9176, + "theoretical_loss": 3.7645659894796903, + "tokens_seen": 727842816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936710130391173, + "loss": 2.7348, + "theoretical_loss": 3.7645318128280865, + "tokens_seen": 727908352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039366098294884655, + "loss": 2.9764, + "theoretical_loss": 3.7644976401148638, + "tokens_seen": 727973888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039365095285857573, + "loss": 2.6341, + "theoretical_loss": 3.764463471339213, + "tokens_seen": 728039424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936409227683049, + "loss": 2.823, + "theoretical_loss": 3.7644293065003263, + "tokens_seen": 728104960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936308926780341, + "loss": 2.6593, + "theoretical_loss": 3.764395145597396, + "tokens_seen": 728170496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039362086258776333, + "loss": 2.7037, + "theoretical_loss": 3.7643609886296145, + "tokens_seen": 728236032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039361083249749246, + "loss": 2.6955, + "theoretical_loss": 3.764326835596174, + "tokens_seen": 728301568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936008024072217, + "loss": 2.6841, + "theoretical_loss": 3.7642926864962685, + "tokens_seen": 728367104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935907723169508, + "loss": 2.8496, + "theoretical_loss": 3.76425854132909, + "tokens_seen": 728432640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039358074222668006, + "loss": 2.7119, + "theoretical_loss": 3.7642244000938314, + "tokens_seen": 728498176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039357071213640924, + "loss": 2.7208, + "theoretical_loss": 3.764190262789688, + "tokens_seen": 728563712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935606820461384, + "loss": 2.5211, + "theoretical_loss": 3.764156129415852, + "tokens_seen": 728629248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935506519558676, + "loss": 2.6906, + "theoretical_loss": 3.7641219999715174, + "tokens_seen": 728694784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935406218655968, + "loss": 2.8027, + "theoretical_loss": 3.76408787445588, + "tokens_seen": 728760320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039353059177532596, + "loss": 2.7496, + "theoretical_loss": 3.7640537528681333, + "tokens_seen": 728825856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935205616850552, + "loss": 2.7888, + "theoretical_loss": 3.7640196352074726, + "tokens_seen": 728891392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935105315947843, + "loss": 2.8002, + "theoretical_loss": 3.7639855214730917, + "tokens_seen": 728956928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039350050150451356, + "loss": 2.8055, + "theoretical_loss": 3.763951411664187, + "tokens_seen": 729022464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 410670, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.842893123626709, + "objective/train/theoretical_loss": 3.7639173057799535, + "objective/train/tokens_used": 749548000, + "theoretical_loss": 3.7639173057799535, + "tokens_seen": 729088000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934904714142427, + "loss": 2.8283, + "theoretical_loss": 3.7639173057799535, + "tokens_seen": 729088000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934804413239719, + "loss": 2.8309, + "theoretical_loss": 3.7638832038195877, + "tokens_seen": 729153536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934704112337011, + "loss": 2.8713, + "theoretical_loss": 3.763849105782284, + "tokens_seen": 729219072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934603811434303, + "loss": 2.6372, + "theoretical_loss": 3.7638150116672398, + "tokens_seen": 729284608 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039345035105315946, + "loss": 2.6612, + "theoretical_loss": 3.763780921473651, + "tokens_seen": 729350144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934403209628887, + "loss": 2.7433, + "theoretical_loss": 3.7637468352007146, + "tokens_seen": 729415680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039343029087261783, + "loss": 2.681, + "theoretical_loss": 3.7637127528476277, + "tokens_seen": 729481216 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039342026078234706, + "loss": 2.8188, + "theoretical_loss": 3.763678674413586, + "tokens_seen": 729546752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934102306920762, + "loss": 2.8761, + "theoretical_loss": 3.7636445998977885, + "tokens_seen": 729612288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934002006018054, + "loss": 2.7117, + "theoretical_loss": 3.7636105292994326, + "tokens_seen": 729677824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039339017051153466, + "loss": 3.0125, + "theoretical_loss": 3.7635764626177153, + "tokens_seen": 729743360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933801404212638, + "loss": 2.9, + "theoretical_loss": 3.7635423998518354, + "tokens_seen": 729808896 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393370110330993, + "loss": 3.011, + "theoretical_loss": 3.763508341000991, + "tokens_seen": 729874432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039336008024072215, + "loss": 2.765, + "theoretical_loss": 3.76347428606438, + "tokens_seen": 729939968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933500501504514, + "loss": 2.8786, + "theoretical_loss": 3.7634402350412026, + "tokens_seen": 730005504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039334002006018057, + "loss": 2.7195, + "theoretical_loss": 3.7634061879306566, + "tokens_seen": 730071040 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039332998996990975, + "loss": 2.8264, + "theoretical_loss": 3.7633721447319415, + "tokens_seen": 730136576 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039331995987963893, + "loss": 2.8929, + "theoretical_loss": 3.763338105444257, + "tokens_seen": 730202112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039330992978936816, + "loss": 2.9521, + "theoretical_loss": 3.763304070066803, + "tokens_seen": 730267648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932998996990973, + "loss": 2.9408, + "theoretical_loss": 3.7632700385987787, + "tokens_seen": 730333184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932898696088265, + "loss": 2.7265, + "theoretical_loss": 3.7632360110393854, + "tokens_seen": 730398720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039327983951855565, + "loss": 2.6464, + "theoretical_loss": 3.7632019873878226, + "tokens_seen": 730464256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932698094282849, + "loss": 2.8734, + "theoretical_loss": 3.763167967643292, + "tokens_seen": 730529792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039325977933801407, + "loss": 2.8426, + "theoretical_loss": 3.763133951804993, + "tokens_seen": 730595328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039324974924774325, + "loss": 2.6467, + "theoretical_loss": 3.763099939872128, + "tokens_seen": 730660864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 411160, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.812192678451538, + "objective/train/theoretical_loss": 3.763065931843898, + "objective/train/tokens_used": 751186400, + "theoretical_loss": 3.763065931843898, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039323971915747243, + "loss": 2.8428, + "theoretical_loss": 3.763065931843898, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932296890672016, + "loss": 2.6838, + "theoretical_loss": 3.7630319277195046, + "tokens_seen": 730791936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932196589769308, + "loss": 3.172, + "theoretical_loss": 3.7629979274981498, + "tokens_seen": 730857472 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039320962888666003, + "loss": 2.6505, + "theoretical_loss": 3.7629639311790353, + "tokens_seen": 730923008 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039319959879638916, + "loss": 2.8953, + "theoretical_loss": 3.762929938761363, + "tokens_seen": 730988544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931895687061184, + "loss": 2.8217, + "theoretical_loss": 3.762895950244337, + "tokens_seen": 731054080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931795386158475, + "loss": 2.6984, + "theoretical_loss": 3.7628619656271587, + "tokens_seen": 731119616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039316950852557675, + "loss": 2.958, + "theoretical_loss": 3.7628279849090314, + "tokens_seen": 731185152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039315947843530593, + "loss": 2.8194, + "theoretical_loss": 3.762794008089158, + "tokens_seen": 731250688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931494483450351, + "loss": 2.8385, + "theoretical_loss": 3.7627600351667434, + "tokens_seen": 731316224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931394182547643, + "loss": 2.7625, + "theoretical_loss": 3.7627260661409903, + "tokens_seen": 731381760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039312938816449353, + "loss": 2.675, + "theoretical_loss": 3.762692101011103, + "tokens_seen": 731447296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039311935807422266, + "loss": 2.8194, + "theoretical_loss": 3.762658139776285, + "tokens_seen": 731512832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931093279839519, + "loss": 2.9804, + "theoretical_loss": 3.762624182435741, + "tokens_seen": 731578368 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393099297893681, + "loss": 2.8233, + "theoretical_loss": 3.7625902289886763, + "tokens_seen": 731643904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039308926780341026, + "loss": 3.0054, + "theoretical_loss": 3.762556279434295, + "tokens_seen": 731709440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039307923771313944, + "loss": 2.9656, + "theoretical_loss": 3.7625223337718023, + "tokens_seen": 731774976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930692076228686, + "loss": 2.7612, + "theoretical_loss": 3.762488392000404, + "tokens_seen": 731840512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930591775325978, + "loss": 2.9062, + "theoretical_loss": 3.7624544541193057, + "tokens_seen": 731906048 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393049147442327, + "loss": 2.5917, + "theoretical_loss": 3.762420520127713, + "tokens_seen": 731971584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039303911735205616, + "loss": 2.8431, + "theoretical_loss": 3.762386590024832, + "tokens_seen": 732037120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930290872617854, + "loss": 2.8048, + "theoretical_loss": 3.7623526638098683, + "tokens_seen": 732102656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930190571715145, + "loss": 2.6144, + "theoretical_loss": 3.76231874148203, + "tokens_seen": 732168192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039300902708124376, + "loss": 2.6214, + "theoretical_loss": 3.762284823040522, + "tokens_seen": 732233728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929989969909729, + "loss": 2.866, + "theoretical_loss": 3.762250908484553, + "tokens_seen": 732299264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 412414, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.900200366973877, + "objective/train/theoretical_loss": 3.7622169978133293, + "objective/train/tokens_used": 752824800, + "theoretical_loss": 3.7622169978133293, + "tokens_seen": 732364800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929889669007021, + "loss": 3.019, + "theoretical_loss": 3.7622169978133293, + "tokens_seen": 732364800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929789368104313, + "loss": 2.8027, + "theoretical_loss": 3.762183091026058, + "tokens_seen": 732430336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929689067201605, + "loss": 2.7653, + "theoretical_loss": 3.7621491881219478, + "tokens_seen": 732495872 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039295887662988967, + "loss": 2.5643, + "theoretical_loss": 3.7621152891002057, + "tokens_seen": 732561408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929488465396189, + "loss": 2.5249, + "theoretical_loss": 3.7620813939600404, + "tokens_seen": 732626944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039293881644934803, + "loss": 2.7563, + "theoretical_loss": 3.76204750270066, + "tokens_seen": 732692480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039292878635907726, + "loss": 2.6964, + "theoretical_loss": 3.7620136153212735, + "tokens_seen": 732758016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929187562688064, + "loss": 2.9321, + "theoretical_loss": 3.7619797318210892, + "tokens_seen": 732823552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929087261785356, + "loss": 2.9943, + "theoretical_loss": 3.7619458521993168, + "tokens_seen": 732889088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928986960882648, + "loss": 3.0453, + "theoretical_loss": 3.761911976455165, + "tokens_seen": 732954624 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392888665997994, + "loss": 2.7816, + "theoretical_loss": 3.7618781045878436, + "tokens_seen": 733020160 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039287863590772317, + "loss": 2.9523, + "theoretical_loss": 3.7618442365965628, + "tokens_seen": 733085696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039286860581745235, + "loss": 2.7939, + "theoretical_loss": 3.7618103724805314, + "tokens_seen": 733151232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039285857572718153, + "loss": 2.8391, + "theoretical_loss": 3.7617765122389613, + "tokens_seen": 733216768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039284854563691077, + "loss": 2.8972, + "theoretical_loss": 3.7617426558710614, + "tokens_seen": 733282304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928385155466399, + "loss": 2.7244, + "theoretical_loss": 3.7617088033760435, + "tokens_seen": 733347840 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039282848545636913, + "loss": 2.5487, + "theoretical_loss": 3.7616749547531185, + "tokens_seen": 733413376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039281845536609825, + "loss": 2.4826, + "theoretical_loss": 3.7616411100014964, + "tokens_seen": 733478912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928084252758275, + "loss": 2.7962, + "theoretical_loss": 3.7616072691203897, + "tokens_seen": 733544448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039279839518555667, + "loss": 2.9535, + "theoretical_loss": 3.76157343210901, + "tokens_seen": 733609984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039278836509528585, + "loss": 2.9491, + "theoretical_loss": 3.7615395989665688, + "tokens_seen": 733675520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039277833500501503, + "loss": 2.9444, + "theoretical_loss": 3.7615057696922785, + "tokens_seen": 733741056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039276830491474427, + "loss": 2.8252, + "theoretical_loss": 3.7614719442853506, + "tokens_seen": 733806592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927582748244734, + "loss": 2.7134, + "theoretical_loss": 3.7614381227449982, + "tokens_seen": 733872128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039274824473420263, + "loss": 2.8441, + "theoretical_loss": 3.7614043050704344, + "tokens_seen": 733937664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 413470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.670003652572632, + "objective/train/theoretical_loss": 3.7613704912608723, + "objective/train/tokens_used": 754463200, + "theoretical_loss": 3.7613704912608723, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039273821464393176, + "loss": 2.9001, + "theoretical_loss": 3.7613704912608723, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392728184553661, + "loss": 2.7337, + "theoretical_loss": 3.761336681315524, + "tokens_seen": 734068736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927181544633902, + "loss": 2.5806, + "theoretical_loss": 3.7613028752336044, + "tokens_seen": 734134272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039270812437311936, + "loss": 2.8466, + "theoretical_loss": 3.761269073014326, + "tokens_seen": 734199808 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039269809428284854, + "loss": 2.7271, + "theoretical_loss": 3.7612352746569035, + "tokens_seen": 734265344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926880641925777, + "loss": 2.8845, + "theoretical_loss": 3.761201480160551, + "tokens_seen": 734330880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926780341023069, + "loss": 2.6129, + "theoretical_loss": 3.7611676895244828, + "tokens_seen": 734396416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039266800401203613, + "loss": 2.646, + "theoretical_loss": 3.761133902747913, + "tokens_seen": 734461952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039265797392176526, + "loss": 2.6668, + "theoretical_loss": 3.7611001198300578, + "tokens_seen": 734527488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926479438314945, + "loss": 2.625, + "theoretical_loss": 3.7610663407701304, + "tokens_seen": 734593024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039263791374122373, + "loss": 2.9871, + "theoretical_loss": 3.7610325655673478, + "tokens_seen": 734658560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039262788365095286, + "loss": 2.8132, + "theoretical_loss": 3.7609987942209244, + "tokens_seen": 734724096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926178535606821, + "loss": 2.7216, + "theoretical_loss": 3.7609650267300765, + "tokens_seen": 734789632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926078234704112, + "loss": 2.7455, + "theoretical_loss": 3.7609312630940197, + "tokens_seen": 734855168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039259779338014046, + "loss": 2.6653, + "theoretical_loss": 3.7608975033119707, + "tokens_seen": 734920704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039258776328986964, + "loss": 2.6471, + "theoretical_loss": 3.760863747383146, + "tokens_seen": 734986240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925777331995988, + "loss": 2.8097, + "theoretical_loss": 3.7608299953067625, + "tokens_seen": 735051776 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392567703109328, + "loss": 2.8716, + "theoretical_loss": 3.760796247082036, + "tokens_seen": 735117312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925576730190572, + "loss": 2.8082, + "theoretical_loss": 3.760762502708185, + "tokens_seen": 735182848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039254764292878636, + "loss": 2.6588, + "theoretical_loss": 3.760728762184425, + "tokens_seen": 735248384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925376128385156, + "loss": 2.731, + "theoretical_loss": 3.7606950255099765, + "tokens_seen": 735313920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925275827482447, + "loss": 2.8463, + "theoretical_loss": 3.7606612926840546, + "tokens_seen": 735379456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039251755265797396, + "loss": 2.6797, + "theoretical_loss": 3.760627563705879, + "tokens_seen": 735444992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925075225677031, + "loss": 2.7702, + "theoretical_loss": 3.7605938385746667, + "tokens_seen": 735510528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924974924774323, + "loss": 2.8931, + "theoretical_loss": 3.7605601172896366, + "tokens_seen": 735576064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 413970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.987671375274658, + "objective/train/theoretical_loss": 3.7605263998500087, + "objective/train/tokens_used": 756101600, + "theoretical_loss": 3.7605263998500087, + "tokens_seen": 735641600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924874623871615, + "loss": 2.9723, + "theoretical_loss": 3.7605263998500087, + "tokens_seen": 735641600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924774322968907, + "loss": 2.759, + "theoretical_loss": 3.7604926862550005, + "tokens_seen": 735707136 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039246740220661987, + "loss": 3.0025, + "theoretical_loss": 3.7604589765038323, + "tokens_seen": 735772672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924573721163491, + "loss": 2.7319, + "theoretical_loss": 3.7604252705957224, + "tokens_seen": 735838208 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039244734202607823, + "loss": 2.8027, + "theoretical_loss": 3.7603915685298914, + "tokens_seen": 735903744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039243731193580746, + "loss": 2.7283, + "theoretical_loss": 3.7603578703055587, + "tokens_seen": 735969280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924272818455366, + "loss": 2.699, + "theoretical_loss": 3.7603241759219443, + "tokens_seen": 736034816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924172517552658, + "loss": 2.9509, + "theoretical_loss": 3.7602904853782695, + "tokens_seen": 736100352 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392407221664995, + "loss": 2.468, + "theoretical_loss": 3.7602567986737534, + "tokens_seen": 736165888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923971915747242, + "loss": 2.6287, + "theoretical_loss": 3.760223115807618, + "tokens_seen": 736231424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039238716148445337, + "loss": 2.8342, + "theoretical_loss": 3.7601894367790836, + "tokens_seen": 736296960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039237713139418255, + "loss": 2.7653, + "theoretical_loss": 3.760155761587372, + "tokens_seen": 736362496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039236710130391173, + "loss": 3.1294, + "theoretical_loss": 3.760122090231704, + "tokens_seen": 736428032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039235707121364097, + "loss": 2.7462, + "theoretical_loss": 3.760088422711302, + "tokens_seen": 736493568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923470411233701, + "loss": 2.7666, + "theoretical_loss": 3.7600547590253877, + "tokens_seen": 736559104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039233701103309933, + "loss": 2.5778, + "theoretical_loss": 3.7600210991731826, + "tokens_seen": 736624640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039232698094282846, + "loss": 2.6942, + "theoretical_loss": 3.75998744315391, + "tokens_seen": 736690176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923169508525577, + "loss": 2.8405, + "theoretical_loss": 3.759953790966793, + "tokens_seen": 736755712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039230692076228687, + "loss": 2.7331, + "theoretical_loss": 3.759920142611053, + "tokens_seen": 736821248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039229689067201605, + "loss": 2.8218, + "theoretical_loss": 3.7598864980859132, + "tokens_seen": 736886784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039228686058174523, + "loss": 2.6316, + "theoretical_loss": 3.759852857390598, + "tokens_seen": 736952320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039227683049147447, + "loss": 2.8765, + "theoretical_loss": 3.75981922052433, + "tokens_seen": 737017856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922668004012036, + "loss": 2.7292, + "theoretical_loss": 3.7597855874863333, + "tokens_seen": 737083392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039225677031093283, + "loss": 2.7402, + "theoretical_loss": 3.7597519582758325, + "tokens_seen": 737148928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039224674022066196, + "loss": 2.6244, + "theoretical_loss": 3.7597183328920507, + "tokens_seen": 737214464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 414896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8144378662109375, + "objective/train/theoretical_loss": 3.759684711334213, + "objective/train/tokens_used": 757740000, + "theoretical_loss": 3.759684711334213, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922367101303912, + "loss": 2.7107, + "theoretical_loss": 3.759684711334213, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922266800401204, + "loss": 2.8554, + "theoretical_loss": 3.7596510936015433, + "tokens_seen": 737345536 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039221664994984956, + "loss": 2.7267, + "theoretical_loss": 3.759617479693267, + "tokens_seen": 737411072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039220661985957874, + "loss": 2.9452, + "theoretical_loss": 3.75958386960861, + "tokens_seen": 737476608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921965897693079, + "loss": 2.7452, + "theoretical_loss": 3.759550263346796, + "tokens_seen": 737542144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921865596790371, + "loss": 3.0845, + "theoretical_loss": 3.759516660907052, + "tokens_seen": 737607680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039217652958876633, + "loss": 2.5286, + "theoretical_loss": 3.759483062288603, + "tokens_seen": 737673216 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039216649949849546, + "loss": 2.8046, + "theoretical_loss": 3.759449467490675, + "tokens_seen": 737738752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921564694082247, + "loss": 2.9167, + "theoretical_loss": 3.7594158765124943, + "tokens_seen": 737804288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921464393179539, + "loss": 2.9723, + "theoretical_loss": 3.759382289353288, + "tokens_seen": 737869824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039213640922768306, + "loss": 2.7637, + "theoretical_loss": 3.759348706012282, + "tokens_seen": 737935360 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039212637913741224, + "loss": 2.8318, + "theoretical_loss": 3.759315126488704, + "tokens_seen": 738000896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921163490471414, + "loss": 2.885, + "theoretical_loss": 3.75928155078178, + "tokens_seen": 738066432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921063189568706, + "loss": 2.8873, + "theoretical_loss": 3.7592479788907376, + "tokens_seen": 738131968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039209628886659984, + "loss": 2.8126, + "theoretical_loss": 3.7592144108148053, + "tokens_seen": 738197504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039208625877632896, + "loss": 2.7385, + "theoretical_loss": 3.7591808465532104, + "tokens_seen": 738263040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920762286860582, + "loss": 2.6291, + "theoretical_loss": 3.759147286105181, + "tokens_seen": 738328576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920661985957873, + "loss": 2.9438, + "theoretical_loss": 3.7591137294699446, + "tokens_seen": 738394112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039205616850551656, + "loss": 2.8742, + "theoretical_loss": 3.759080176646731, + "tokens_seen": 738459648 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039204613841524574, + "loss": 2.5495, + "theoretical_loss": 3.759046627634768, + "tokens_seen": 738525184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920361083249749, + "loss": 2.5249, + "theoretical_loss": 3.759013082433285, + "tokens_seen": 738590720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920260782347041, + "loss": 2.7102, + "theoretical_loss": 3.7589795410415103, + "tokens_seen": 738656256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920160481444333, + "loss": 2.9134, + "theoretical_loss": 3.758946003458675, + "tokens_seen": 738721792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039200601805416247, + "loss": 2.8764, + "theoretical_loss": 3.758912469684007, + "tokens_seen": 738787328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919959879638917, + "loss": 2.7331, + "theoretical_loss": 3.7588789397167366, + "tokens_seen": 738852864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 415468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.898322820663452, + "objective/train/theoretical_loss": 3.7588454135560943, + "objective/train/tokens_used": 759378400, + "theoretical_loss": 3.7588454135560943, + "tokens_seen": 738918400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039198595787362083, + "loss": 2.7734, + "theoretical_loss": 3.7588454135560943, + "tokens_seen": 738918400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039197592778335007, + "loss": 2.7841, + "theoretical_loss": 3.7588118912013098, + "tokens_seen": 738983936 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039196589769307925, + "loss": 2.8882, + "theoretical_loss": 3.758778372651614, + "tokens_seen": 739049472 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039195586760280843, + "loss": 2.8642, + "theoretical_loss": 3.7587448579062377, + "tokens_seen": 739115008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919458375125376, + "loss": 2.7614, + "theoretical_loss": 3.758711346964411, + "tokens_seen": 739180544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919358074222668, + "loss": 2.8015, + "theoretical_loss": 3.7586778398253666, + "tokens_seen": 739246080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039192577733199597, + "loss": 2.893, + "theoretical_loss": 3.7586443364883344, + "tokens_seen": 739311616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919157472417252, + "loss": 2.9149, + "theoretical_loss": 3.7586108369525473, + "tokens_seen": 739377152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039190571715145433, + "loss": 2.5778, + "theoretical_loss": 3.7585773412172356, + "tokens_seen": 739442688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039189568706118357, + "loss": 2.9246, + "theoretical_loss": 3.758543849281633, + "tokens_seen": 739508224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039188565697091275, + "loss": 2.844, + "theoretical_loss": 3.758510361144971, + "tokens_seen": 739573760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039187562688064193, + "loss": 2.7873, + "theoretical_loss": 3.7584768768064816, + "tokens_seen": 739639296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039186559679037117, + "loss": 2.9233, + "theoretical_loss": 3.758443396265399, + "tokens_seen": 739704832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918555667001003, + "loss": 2.7354, + "theoretical_loss": 3.758409919520955, + "tokens_seen": 739770368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039184553660982953, + "loss": 2.8651, + "theoretical_loss": 3.7583764465723823, + "tokens_seen": 739835904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039183550651955866, + "loss": 2.791, + "theoretical_loss": 3.758342977418916, + "tokens_seen": 739901440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918254764292879, + "loss": 2.7292, + "theoretical_loss": 3.758309512059788, + "tokens_seen": 739966976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039181544633901707, + "loss": 2.9291, + "theoretical_loss": 3.758276050494233, + "tokens_seen": 740032512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039180541624874625, + "loss": 2.8555, + "theoretical_loss": 3.7582425927214853, + "tokens_seen": 740098048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039179538615847543, + "loss": 2.8255, + "theoretical_loss": 3.7582091387407788, + "tokens_seen": 740163584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039178535606820467, + "loss": 2.6274, + "theoretical_loss": 3.758175688551348, + "tokens_seen": 740229120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917753259779338, + "loss": 2.7021, + "theoretical_loss": 3.758142242152428, + "tokens_seen": 740294656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039176529588766303, + "loss": 2.7977, + "theoretical_loss": 3.758108799543254, + "tokens_seen": 740360192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039175526579739216, + "loss": 3.0315, + "theoretical_loss": 3.7580753607230593, + "tokens_seen": 740425728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917452357071214, + "loss": 2.8211, + "theoretical_loss": 3.7580419256910815, + "tokens_seen": 740491264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 416744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.713137626647949, + "objective/train/theoretical_loss": 3.7580084944465555, + "objective/train/tokens_used": 761016800, + "theoretical_loss": 3.7580084944465555, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917352056168506, + "loss": 2.8892, + "theoretical_loss": 3.7580084944465555, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039172517552657976, + "loss": 2.933, + "theoretical_loss": 3.757975066988717, + "tokens_seen": 740622336 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039171514543630894, + "loss": 2.7738, + "theoretical_loss": 3.7579416433168022, + "tokens_seen": 740687872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917051153460381, + "loss": 3.0515, + "theoretical_loss": 3.757908223430047, + "tokens_seen": 740753408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916950852557673, + "loss": 2.7393, + "theoretical_loss": 3.7578748073276884, + "tokens_seen": 740818944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039168505516549653, + "loss": 2.5972, + "theoretical_loss": 3.7578413950089633, + "tokens_seen": 740884480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039167502507522566, + "loss": 2.8358, + "theoretical_loss": 3.757807986473108, + "tokens_seen": 740950016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916649949849549, + "loss": 2.6884, + "theoretical_loss": 3.75777458171936, + "tokens_seen": 741015552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916549648946841, + "loss": 2.9793, + "theoretical_loss": 3.757741180746957, + "tokens_seen": 741081088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039164493480441326, + "loss": 2.938, + "theoretical_loss": 3.7577077835551362, + "tokens_seen": 741146624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039163490471414244, + "loss": 2.772, + "theoretical_loss": 3.7576743901431353, + "tokens_seen": 741212160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916248746238716, + "loss": 2.7569, + "theoretical_loss": 3.7576410005101932, + "tokens_seen": 741277696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916148445336008, + "loss": 2.74, + "theoretical_loss": 3.757607614655547, + "tokens_seen": 741343232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039160481444333004, + "loss": 2.8583, + "theoretical_loss": 3.7575742325784365, + "tokens_seen": 741408768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039159478435305916, + "loss": 2.9713, + "theoretical_loss": 3.7575408542780995, + "tokens_seen": 741474304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915847542627884, + "loss": 2.6112, + "theoretical_loss": 3.757507479753775, + "tokens_seen": 741539840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915747241725175, + "loss": 2.6381, + "theoretical_loss": 3.7574741090047024, + "tokens_seen": 741605376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039156469408224676, + "loss": 2.6271, + "theoretical_loss": 3.7574407420301217, + "tokens_seen": 741670912 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039155466399197594, + "loss": 2.7429, + "theoretical_loss": 3.7574073788292712, + "tokens_seen": 741736448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915446339017051, + "loss": 2.7142, + "theoretical_loss": 3.7573740194013916, + "tokens_seen": 741801984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915346038114343, + "loss": 2.7973, + "theoretical_loss": 3.7573406637457225, + "tokens_seen": 741867520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915245737211635, + "loss": 2.6915, + "theoretical_loss": 3.757307311861505, + "tokens_seen": 741933056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039151454363089267, + "loss": 2.9268, + "theoretical_loss": 3.757273963747979, + "tokens_seen": 741998592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915045135406219, + "loss": 2.5838, + "theoretical_loss": 3.7572406194043846, + "tokens_seen": 742064128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039149448345035103, + "loss": 2.7691, + "theoretical_loss": 3.7572072788299637, + "tokens_seen": 742129664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 417361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1539878845214844, + "objective/train/theoretical_loss": 3.7571739420239574, + "objective/train/tokens_used": 762655200, + "theoretical_loss": 3.7571739420239574, + "tokens_seen": 742195200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039148445336008027, + "loss": 2.8494, + "theoretical_loss": 3.7571739420239574, + "tokens_seen": 742195200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039147442326980945, + "loss": 2.7746, + "theoretical_loss": 3.757140608985606, + "tokens_seen": 742260736 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039146439317953863, + "loss": 2.8907, + "theoretical_loss": 3.7571072797141523, + "tokens_seen": 742326272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914543630892678, + "loss": 2.8257, + "theoretical_loss": 3.7570739542088374, + "tokens_seen": 742391808 + }, + { + "epoch": 0.22, + "learning_rate": 0.000391444332998997, + "loss": 2.9461, + "theoretical_loss": 3.757040632468904, + "tokens_seen": 742457344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039143430290872617, + "loss": 2.7556, + "theoretical_loss": 3.7570073144935936, + "tokens_seen": 742522880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003914242728184554, + "loss": 2.8871, + "theoretical_loss": 3.7569740002821495, + "tokens_seen": 742588416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039141424272818453, + "loss": 2.8353, + "theoretical_loss": 3.7569406898338134, + "tokens_seen": 742653952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039140421263791377, + "loss": 2.7706, + "theoretical_loss": 3.756907383147829, + "tokens_seen": 742719488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913941825476429, + "loss": 2.7584, + "theoretical_loss": 3.756874080223439, + "tokens_seen": 742785024 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039138415245737213, + "loss": 2.802, + "theoretical_loss": 3.7568407810598865, + "tokens_seen": 742850560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913741223671013, + "loss": 2.6823, + "theoretical_loss": 3.756807485656416, + "tokens_seen": 742916096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913640922768305, + "loss": 2.8226, + "theoretical_loss": 3.7567741940122703, + "tokens_seen": 742981632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913540621865597, + "loss": 2.7535, + "theoretical_loss": 3.7567409061266943, + "tokens_seen": 743047168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039134403209628886, + "loss": 2.5167, + "theoretical_loss": 3.756707621998931, + "tokens_seen": 743112704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039133400200601804, + "loss": 2.7885, + "theoretical_loss": 3.7566743416282264, + "tokens_seen": 743178240 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039132397191574727, + "loss": 2.6626, + "theoretical_loss": 3.7566410650138238, + "tokens_seen": 743243776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913139418254764, + "loss": 2.6599, + "theoretical_loss": 3.756607792154969, + "tokens_seen": 743309312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039130391173520563, + "loss": 2.755, + "theoretical_loss": 3.7565745230509062, + "tokens_seen": 743374848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912938816449348, + "loss": 2.7896, + "theoretical_loss": 3.7565412577008814, + "tokens_seen": 743440384 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391283851554664, + "loss": 2.5392, + "theoretical_loss": 3.7565079961041397, + "tokens_seen": 743505920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912738214643932, + "loss": 2.7874, + "theoretical_loss": 3.7564747382599273, + "tokens_seen": 743571456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039126379137412236, + "loss": 2.5402, + "theoretical_loss": 3.75644148416749, + "tokens_seen": 743636992 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039125376128385154, + "loss": 2.6678, + "theoretical_loss": 3.7564082338260736, + "tokens_seen": 743702528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912437311935808, + "loss": 2.7886, + "theoretical_loss": 3.756374987234925, + "tokens_seen": 743768064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 418521, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7649760246276855, + "objective/train/theoretical_loss": 3.7563417443932905, + "objective/train/tokens_used": 764293600, + "theoretical_loss": 3.7563417443932905, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912337011033099, + "loss": 2.9361, + "theoretical_loss": 3.7563417443932905, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039122367101303914, + "loss": 2.9609, + "theoretical_loss": 3.7563085053004173, + "tokens_seen": 743899136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039121364092276826, + "loss": 2.7547, + "theoretical_loss": 3.7562752699555517, + "tokens_seen": 743964672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912036108324975, + "loss": 2.5979, + "theoretical_loss": 3.756242038357942, + "tokens_seen": 744030208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911935807422267, + "loss": 2.9392, + "theoretical_loss": 3.756208810506835, + "tokens_seen": 744095744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039118355065195586, + "loss": 2.6047, + "theoretical_loss": 3.7561755864014787, + "tokens_seen": 744161280 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039117352056168504, + "loss": 2.8347, + "theoretical_loss": 3.756142366041121, + "tokens_seen": 744226816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911634904714143, + "loss": 2.7851, + "theoretical_loss": 3.7561091494250096, + "tokens_seen": 744292352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911534603811434, + "loss": 2.7388, + "theoretical_loss": 3.756075936552394, + "tokens_seen": 744357888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039114343029087264, + "loss": 2.7189, + "theoretical_loss": 3.756042727422521, + "tokens_seen": 744423424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911334002006018, + "loss": 2.6181, + "theoretical_loss": 3.756009522034641, + "tokens_seen": 744488960 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391123370110331, + "loss": 2.7899, + "theoretical_loss": 3.755976320388002, + "tokens_seen": 744554496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039111334002006024, + "loss": 2.7496, + "theoretical_loss": 3.755943122481854, + "tokens_seen": 744620032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039110330992978936, + "loss": 2.9665, + "theoretical_loss": 3.755909928315446, + "tokens_seen": 744685568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910932798395186, + "loss": 2.9362, + "theoretical_loss": 3.7558767378880273, + "tokens_seen": 744751104 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039108324974924773, + "loss": 2.5904, + "theoretical_loss": 3.755843551198849, + "tokens_seen": 744816640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039107321965897696, + "loss": 2.6235, + "theoretical_loss": 3.7558103682471593, + "tokens_seen": 744882176 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039106318956870614, + "loss": 2.6325, + "theoretical_loss": 3.7557771890322105, + "tokens_seen": 744947712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910531594784353, + "loss": 2.7591, + "theoretical_loss": 3.755744013553252, + "tokens_seen": 745013248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910431293881645, + "loss": 2.6799, + "theoretical_loss": 3.755710841809534, + "tokens_seen": 745078784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910330992978937, + "loss": 2.7825, + "theoretical_loss": 3.755677673800309, + "tokens_seen": 745144320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039102306920762287, + "loss": 2.7031, + "theoretical_loss": 3.755644509524827, + "tokens_seen": 745209856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910130391173521, + "loss": 2.6609, + "theoretical_loss": 3.75561134898234, + "tokens_seen": 745275392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039100300902708123, + "loss": 2.5443, + "theoretical_loss": 3.7555781921720994, + "tokens_seen": 745340928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039099297893681047, + "loss": 2.7118, + "theoretical_loss": 3.7555450390933567, + "tokens_seen": 745406464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 419167, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.160205125808716, + "objective/train/theoretical_loss": 3.7555118897453648, + "objective/train/tokens_used": 765932000, + "theoretical_loss": 3.7555118897453648, + "tokens_seen": 745472000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039098294884653965, + "loss": 2.8914, + "theoretical_loss": 3.7555118897453648, + "tokens_seen": 745472000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039097291875626883, + "loss": 2.6525, + "theoretical_loss": 3.755478744127375, + "tokens_seen": 745537536 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390962888665998, + "loss": 2.771, + "theoretical_loss": 3.75544560223864, + "tokens_seen": 745603072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909528585757272, + "loss": 2.7758, + "theoretical_loss": 3.755412464078413, + "tokens_seen": 745668608 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039094282848545637, + "loss": 2.7658, + "theoretical_loss": 3.755379329645946, + "tokens_seen": 745734144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909327983951856, + "loss": 2.8785, + "theoretical_loss": 3.7553461989404933, + "tokens_seen": 745799680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039092276830491473, + "loss": 2.7476, + "theoretical_loss": 3.7553130719613073, + "tokens_seen": 745865216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039091273821464397, + "loss": 2.5613, + "theoretical_loss": 3.755279948707642, + "tokens_seen": 745930752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909027081243731, + "loss": 2.7278, + "theoretical_loss": 3.7552468291787506, + "tokens_seen": 745996288 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039089267803410233, + "loss": 2.4587, + "theoretical_loss": 3.7552137133738874, + "tokens_seen": 746061824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908826479438315, + "loss": 2.873, + "theoretical_loss": 3.755180601292307, + "tokens_seen": 746127360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908726178535607, + "loss": 2.8608, + "theoretical_loss": 3.7551474929332636, + "tokens_seen": 746192896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908625877632899, + "loss": 2.9343, + "theoretical_loss": 3.755114388296011, + "tokens_seen": 746258432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039085255767301906, + "loss": 2.5345, + "theoretical_loss": 3.7550812873798054, + "tokens_seen": 746323968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039084252758274824, + "loss": 2.5794, + "theoretical_loss": 3.755048190183901, + "tokens_seen": 746389504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039083249749247747, + "loss": 2.5169, + "theoretical_loss": 3.755015096707553, + "tokens_seen": 746455040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908224674022066, + "loss": 2.7485, + "theoretical_loss": 3.754982006950017, + "tokens_seen": 746520576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039081243731193583, + "loss": 2.6492, + "theoretical_loss": 3.7549489209105493, + "tokens_seen": 746586112 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390802407221665, + "loss": 2.4441, + "theoretical_loss": 3.754915838588405, + "tokens_seen": 746651648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907923771313942, + "loss": 2.6574, + "theoretical_loss": 3.7548827599828405, + "tokens_seen": 746717184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907823470411234, + "loss": 2.8731, + "theoretical_loss": 3.754849685093112, + "tokens_seen": 746782720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039077231695085256, + "loss": 2.6081, + "theoretical_loss": 3.754816613918476, + "tokens_seen": 746848256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039076228686058174, + "loss": 2.744, + "theoretical_loss": 3.7547835464581896, + "tokens_seen": 746913792 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390752256770311, + "loss": 2.9731, + "theoretical_loss": 3.7547504827115104, + "tokens_seen": 746979328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907422266800401, + "loss": 2.9428, + "theoretical_loss": 3.754717422677694, + "tokens_seen": 747044864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 420047, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9569199085235596, + "objective/train/theoretical_loss": 3.754684366355999, + "objective/train/tokens_used": 767570400, + "theoretical_loss": 3.754684366355999, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039073219658976934, + "loss": 2.6868, + "theoretical_loss": 3.754684366355999, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039072216649949846, + "loss": 2.8481, + "theoretical_loss": 3.7546513137456823, + "tokens_seen": 747175936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907121364092277, + "loss": 2.6763, + "theoretical_loss": 3.7546182648460023, + "tokens_seen": 747241472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907021063189569, + "loss": 2.541, + "theoretical_loss": 3.754585219656217, + "tokens_seen": 747307008 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039069207622868606, + "loss": 2.811, + "theoretical_loss": 3.754552178175584, + "tokens_seen": 747372544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039068204613841524, + "loss": 2.8739, + "theoretical_loss": 3.7545191404033624, + "tokens_seen": 747438080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906720160481445, + "loss": 2.7239, + "theoretical_loss": 3.7544861063388106, + "tokens_seen": 747503616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906619859578736, + "loss": 2.7788, + "theoretical_loss": 3.7544530759811883, + "tokens_seen": 747569152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039065195586760284, + "loss": 2.7612, + "theoretical_loss": 3.7544200493297533, + "tokens_seen": 747634688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039064192577733197, + "loss": 2.4776, + "theoretical_loss": 3.7543870263837658, + "tokens_seen": 747700224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906318956870612, + "loss": 2.6039, + "theoretical_loss": 3.754354007142485, + "tokens_seen": 747765760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906218655967904, + "loss": 2.8779, + "theoretical_loss": 3.754320991605171, + "tokens_seen": 747831296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039061183550651957, + "loss": 2.6714, + "theoretical_loss": 3.754287979771083, + "tokens_seen": 747896832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039060180541624875, + "loss": 2.8675, + "theoretical_loss": 3.754254971639482, + "tokens_seen": 747962368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039059177532597793, + "loss": 2.8031, + "theoretical_loss": 3.754221967209628, + "tokens_seen": 748027904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905817452357071, + "loss": 2.5851, + "theoretical_loss": 3.754188966480782, + "tokens_seen": 748093440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039057171514543634, + "loss": 2.6848, + "theoretical_loss": 3.754155969452204, + "tokens_seen": 748158976 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039056168505516547, + "loss": 2.8457, + "theoretical_loss": 3.7541229761231563, + "tokens_seen": 748224512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905516549648947, + "loss": 2.8551, + "theoretical_loss": 3.7540899864928985, + "tokens_seen": 748290048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039054162487462383, + "loss": 2.9909, + "theoretical_loss": 3.7540570005606932, + "tokens_seen": 748355584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039053159478435307, + "loss": 2.5867, + "theoretical_loss": 3.754024018325802, + "tokens_seen": 748421120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039052156469408225, + "loss": 2.7651, + "theoretical_loss": 3.7539910397874863, + "tokens_seen": 748486656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039051153460381143, + "loss": 2.7933, + "theoretical_loss": 3.7539580649450084, + "tokens_seen": 748552192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905015045135406, + "loss": 2.9215, + "theoretical_loss": 3.7539250937976307, + "tokens_seen": 748617728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039049147442326985, + "loss": 2.6618, + "theoretical_loss": 3.7538921263446157, + "tokens_seen": 748683264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 421176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7754266262054443, + "objective/train/theoretical_loss": 3.753859162585226, + "objective/train/tokens_used": 769208800, + "theoretical_loss": 3.753859162585226, + "tokens_seen": 748748800 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390481444332999, + "loss": 2.8535, + "theoretical_loss": 3.753859162585226, + "tokens_seen": 748748800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904714142427282, + "loss": 2.8332, + "theoretical_loss": 3.7538262025187246, + "tokens_seen": 748814336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039046138415245734, + "loss": 2.6704, + "theoretical_loss": 3.753793246144374, + "tokens_seen": 748879872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039045135406218657, + "loss": 2.8584, + "theoretical_loss": 3.753760293461439, + "tokens_seen": 748945408 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039044132397191575, + "loss": 2.7273, + "theoretical_loss": 3.7537273444691825, + "tokens_seen": 749010944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039043129388164493, + "loss": 3.0141, + "theoretical_loss": 3.753694399166867, + "tokens_seen": 749076480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904212637913741, + "loss": 2.5281, + "theoretical_loss": 3.7536614575537586, + "tokens_seen": 749142016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904112337011033, + "loss": 2.7591, + "theoretical_loss": 3.75362851962912, + "tokens_seen": 749207552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904012036108325, + "loss": 2.7426, + "theoretical_loss": 3.7535955853922167, + "tokens_seen": 749273088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903911735205617, + "loss": 2.9189, + "theoretical_loss": 3.7535626548423124, + "tokens_seen": 749338624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903811434302909, + "loss": 2.7107, + "theoretical_loss": 3.7535297279786723, + "tokens_seen": 749404160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903711133400201, + "loss": 2.8125, + "theoretical_loss": 3.7534968048005615, + "tokens_seen": 749469696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039036108324974926, + "loss": 2.8113, + "theoretical_loss": 3.7534638853072453, + "tokens_seen": 749535232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039035105315947844, + "loss": 2.7766, + "theoretical_loss": 3.7534309694979884, + "tokens_seen": 749600768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039034102306920767, + "loss": 2.8065, + "theoretical_loss": 3.753398057372058, + "tokens_seen": 749666304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903309929789368, + "loss": 2.6713, + "theoretical_loss": 3.7533651489287188, + "tokens_seen": 749731840 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039032096288866603, + "loss": 2.8039, + "theoretical_loss": 3.753332244167237, + "tokens_seen": 749797376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903109327983952, + "loss": 2.7021, + "theoretical_loss": 3.753299343086879, + "tokens_seen": 749862912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903009027081244, + "loss": 2.6358, + "theoretical_loss": 3.753266445686912, + "tokens_seen": 749928448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902908726178536, + "loss": 2.5634, + "theoretical_loss": 3.753233551966602, + "tokens_seen": 749993984 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039028084252758276, + "loss": 2.717, + "theoretical_loss": 3.7532006619252156, + "tokens_seen": 750059520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039027081243731194, + "loss": 2.8146, + "theoretical_loss": 3.753167775562021, + "tokens_seen": 750125056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902607823470412, + "loss": 2.799, + "theoretical_loss": 3.753134892876285, + "tokens_seen": 750190592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902507522567703, + "loss": 2.648, + "theoretical_loss": 3.753102013867275, + "tokens_seen": 750256128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039024072216649954, + "loss": 2.6237, + "theoretical_loss": 3.753069138534259, + "tokens_seen": 750321664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 421843, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.485619068145752, + "objective/train/theoretical_loss": 3.753036266876505, + "objective/train/tokens_used": 770847200, + "theoretical_loss": 3.753036266876505, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039023069207622866, + "loss": 2.5361, + "theoretical_loss": 3.753036266876505, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902206619859579, + "loss": 2.828, + "theoretical_loss": 3.753003398893281, + "tokens_seen": 750452736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902106318956871, + "loss": 2.6835, + "theoretical_loss": 3.7529705345838558, + "tokens_seen": 750518272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039020060180541626, + "loss": 2.8157, + "theoretical_loss": 3.752937673947498, + "tokens_seen": 750583808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039019057171514544, + "loss": 2.6794, + "theoretical_loss": 3.7529048169834756, + "tokens_seen": 750649344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901805416248747, + "loss": 2.8133, + "theoretical_loss": 3.7528719636910584, + "tokens_seen": 750714880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901705115346038, + "loss": 2.6176, + "theoretical_loss": 3.7528391140695154, + "tokens_seen": 750780416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039016048144433304, + "loss": 2.9973, + "theoretical_loss": 3.7528062681181167, + "tokens_seen": 750845952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039015045135406217, + "loss": 2.8111, + "theoretical_loss": 3.752773425836131, + "tokens_seen": 750911488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901404212637914, + "loss": 2.8781, + "theoretical_loss": 3.7527405872228288, + "tokens_seen": 750977024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901303911735206, + "loss": 2.8973, + "theoretical_loss": 3.75270775227748, + "tokens_seen": 751042560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039012036108324977, + "loss": 2.7338, + "theoretical_loss": 3.7526749209993544, + "tokens_seen": 751108096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039011033099297895, + "loss": 2.7779, + "theoretical_loss": 3.752642093387723, + "tokens_seen": 751173632 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039010030090270813, + "loss": 2.7332, + "theoretical_loss": 3.752609269441857, + "tokens_seen": 751239168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900902708124373, + "loss": 2.6754, + "theoretical_loss": 3.7525764491610265, + "tokens_seen": 751304704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039008024072216654, + "loss": 2.7638, + "theoretical_loss": 3.7525436325445027, + "tokens_seen": 751370240 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039007021063189567, + "loss": 2.9505, + "theoretical_loss": 3.7525108195915573, + "tokens_seen": 751435776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900601805416249, + "loss": 2.5367, + "theoretical_loss": 3.752478010301462, + "tokens_seen": 751501312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039005015045135403, + "loss": 2.679, + "theoretical_loss": 3.7524452046734877, + "tokens_seen": 751566848 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039004012036108327, + "loss": 2.8081, + "theoretical_loss": 3.752412402706907, + "tokens_seen": 751632384 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039003009027081245, + "loss": 2.8126, + "theoretical_loss": 3.7523796044009923, + "tokens_seen": 751697920 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039002006018054163, + "loss": 2.6605, + "theoretical_loss": 3.7523468097550152, + "tokens_seen": 751763456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900100300902708, + "loss": 2.6298, + "theoretical_loss": 3.7523140187682493, + "tokens_seen": 751828992 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039000000000000005, + "loss": 2.7272, + "theoretical_loss": 3.7522812314399663, + "tokens_seen": 751894528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899899699097292, + "loss": 2.9884, + "theoretical_loss": 3.7522484477694396, + "tokens_seen": 751960064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 423030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.662358283996582, + "objective/train/theoretical_loss": 3.7522156677559426, + "objective/train/tokens_used": 772485600, + "theoretical_loss": 3.7522156677559426, + "tokens_seen": 752025600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899799398194584, + "loss": 2.7389, + "theoretical_loss": 3.7522156677559426, + "tokens_seen": 752025600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038996990972918754, + "loss": 2.9452, + "theoretical_loss": 3.752182891398749, + "tokens_seen": 752091136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038995987963891677, + "loss": 2.9664, + "theoretical_loss": 3.752150118697132, + "tokens_seen": 752156672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038994984954864595, + "loss": 2.7687, + "theoretical_loss": 3.752117349650365, + "tokens_seen": 752222208 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038993981945837513, + "loss": 2.9713, + "theoretical_loss": 3.752084584257723, + "tokens_seen": 752287744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899297893681043, + "loss": 2.8341, + "theoretical_loss": 3.75205182251848, + "tokens_seen": 752353280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899197592778335, + "loss": 2.7872, + "theoretical_loss": 3.7520190644319094, + "tokens_seen": 752418816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899097291875627, + "loss": 2.5177, + "theoretical_loss": 3.751986309997287, + "tokens_seen": 752484352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898996990972919, + "loss": 3.0072, + "theoretical_loss": 3.751953559213888, + "tokens_seen": 752549888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038988966900702104, + "loss": 2.7622, + "theoretical_loss": 3.7519208120809857, + "tokens_seen": 752615424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898796389167503, + "loss": 2.8546, + "theoretical_loss": 3.7518880685978573, + "tokens_seen": 752680960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898696088264794, + "loss": 2.6212, + "theoretical_loss": 3.751855328763777, + "tokens_seen": 752746496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038985957873620864, + "loss": 2.4672, + "theoretical_loss": 3.751822592578022, + "tokens_seen": 752812032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898495486459378, + "loss": 2.624, + "theoretical_loss": 3.7517898600398665, + "tokens_seen": 752877568 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389839518555667, + "loss": 2.7594, + "theoretical_loss": 3.7517571311485867, + "tokens_seen": 752943104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898294884653962, + "loss": 2.8784, + "theoretical_loss": 3.7517244059034605, + "tokens_seen": 753008640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898194583751254, + "loss": 2.57, + "theoretical_loss": 3.7516916843037627, + "tokens_seen": 753074176 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038980942828485454, + "loss": 2.69, + "theoretical_loss": 3.751658966348771, + "tokens_seen": 753139712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897993981945838, + "loss": 2.7796, + "theoretical_loss": 3.751626252037762, + "tokens_seen": 753205248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897893681043129, + "loss": 2.8479, + "theoretical_loss": 3.7515935413700134, + "tokens_seen": 753270784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038977933801404214, + "loss": 2.803, + "theoretical_loss": 3.7515608343448017, + "tokens_seen": 753336320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897693079237713, + "loss": 2.6135, + "theoretical_loss": 3.751528130961405, + "tokens_seen": 753401856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897592778335005, + "loss": 2.8919, + "theoretical_loss": 3.7514954312191007, + "tokens_seen": 753467392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897492477432297, + "loss": 2.759, + "theoretical_loss": 3.751462735117167, + "tokens_seen": 753532928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038973921765295886, + "loss": 2.639, + "theoretical_loss": 3.7514300426548823, + "tokens_seen": 753598464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 424346, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4410223960876465, + "objective/train/theoretical_loss": 3.751397353831524, + "objective/train/tokens_used": 774124000, + "theoretical_loss": 3.751397353831524, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038972918756268805, + "loss": 2.5345, + "theoretical_loss": 3.751397353831524, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897191574724173, + "loss": 2.8886, + "theoretical_loss": 3.751364668646372, + "tokens_seen": 753729536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897091273821464, + "loss": 2.5767, + "theoretical_loss": 3.7513319870987045, + "tokens_seen": 753795072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038969909729187564, + "loss": 2.6723, + "theoretical_loss": 3.7512993091878, + "tokens_seen": 753860608 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038968906720160477, + "loss": 2.8139, + "theoretical_loss": 3.751266634912939, + "tokens_seen": 753926144 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389679037111334, + "loss": 2.673, + "theoretical_loss": 3.7512339642734, + "tokens_seen": 753991680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896690070210632, + "loss": 2.4749, + "theoretical_loss": 3.7512012972684623, + "tokens_seen": 754057216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038965897693079237, + "loss": 2.8392, + "theoretical_loss": 3.751168633897406, + "tokens_seen": 754122752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038964894684052155, + "loss": 2.6861, + "theoretical_loss": 3.751135974159511, + "tokens_seen": 754188288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896389167502508, + "loss": 2.6624, + "theoretical_loss": 3.7511033180540583, + "tokens_seen": 754253824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038962888665997997, + "loss": 2.7724, + "theoretical_loss": 3.7510706655803276, + "tokens_seen": 754319360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038961885656970915, + "loss": 2.9919, + "theoretical_loss": 3.7510380167375996, + "tokens_seen": 754384896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038960882647943833, + "loss": 2.6663, + "theoretical_loss": 3.7510053715251557, + "tokens_seen": 754450432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895987963891675, + "loss": 2.778, + "theoretical_loss": 3.750972729942276, + "tokens_seen": 754515968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038958876629889674, + "loss": 2.795, + "theoretical_loss": 3.7509400919882427, + "tokens_seen": 754581504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038957873620862587, + "loss": 2.6879, + "theoretical_loss": 3.7509074576623362, + "tokens_seen": 754647040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895687061183551, + "loss": 2.8031, + "theoretical_loss": 3.7508748269638397, + "tokens_seen": 754712576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038955867602808423, + "loss": 2.7347, + "theoretical_loss": 3.7508421998920336, + "tokens_seen": 754778112 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038954864593781347, + "loss": 2.8483, + "theoretical_loss": 3.7508095764462004, + "tokens_seen": 754843648 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038953861584754265, + "loss": 2.8203, + "theoretical_loss": 3.750776956625623, + "tokens_seen": 754909184 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038952858575727183, + "loss": 2.6643, + "theoretical_loss": 3.7507443404295833, + "tokens_seen": 754974720 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389518555667001, + "loss": 2.8899, + "theoretical_loss": 3.7507117278573636, + "tokens_seen": 755040256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038950852557673025, + "loss": 2.8196, + "theoretical_loss": 3.7506791189082476, + "tokens_seen": 755105792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894984954864594, + "loss": 2.7476, + "theoretical_loss": 3.750646513581518, + "tokens_seen": 755171328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894884653961886, + "loss": 2.5687, + "theoretical_loss": 3.750613911876458, + "tokens_seen": 755236864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 425020, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5594170093536377, + "objective/train/theoretical_loss": 3.750581313792351, + "objective/train/tokens_used": 775762400, + "theoretical_loss": 3.750581313792351, + "tokens_seen": 755302400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038947843530591774, + "loss": 2.419, + "theoretical_loss": 3.750581313792351, + "tokens_seen": 755302400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038946840521564697, + "loss": 2.659, + "theoretical_loss": 3.7505487193284814, + "tokens_seen": 755367936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038945837512537615, + "loss": 2.7292, + "theoretical_loss": 3.750516128484133, + "tokens_seen": 755433472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038944834503510533, + "loss": 2.8103, + "theoretical_loss": 3.7504835412585886, + "tokens_seen": 755499008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894383149448345, + "loss": 2.7047, + "theoretical_loss": 3.7504509576511342, + "tokens_seen": 755564544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894282848545637, + "loss": 2.6846, + "theoretical_loss": 3.7504183776610533, + "tokens_seen": 755630080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894182547642929, + "loss": 2.7696, + "theoretical_loss": 3.7503858012876305, + "tokens_seen": 755695616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894082246740221, + "loss": 2.746, + "theoretical_loss": 3.7503532285301517, + "tokens_seen": 755761152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038939819458375124, + "loss": 2.7698, + "theoretical_loss": 3.7503206593879015, + "tokens_seen": 755826688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893881644934805, + "loss": 2.661, + "theoretical_loss": 3.7502880938601653, + "tokens_seen": 755892224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893781344032096, + "loss": 2.5849, + "theoretical_loss": 3.750255531946228, + "tokens_seen": 755957760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038936810431293884, + "loss": 2.7654, + "theoretical_loss": 3.7502229736453763, + "tokens_seen": 756023296 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389358074222668, + "loss": 2.8642, + "theoretical_loss": 3.7501904189568958, + "tokens_seen": 756088832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893480441323972, + "loss": 2.7808, + "theoretical_loss": 3.7501578678800724, + "tokens_seen": 756154368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893380140421264, + "loss": 2.4493, + "theoretical_loss": 3.750125320414192, + "tokens_seen": 756219904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893279839518556, + "loss": 2.7259, + "theoretical_loss": 3.7500927765585423, + "tokens_seen": 756285440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038931795386158474, + "loss": 2.7156, + "theoretical_loss": 3.75006023631241, + "tokens_seen": 756350976 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389307923771314, + "loss": 2.7651, + "theoretical_loss": 3.7500276996750808, + "tokens_seen": 756416512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892978936810431, + "loss": 2.979, + "theoretical_loss": 3.749995166645843, + "tokens_seen": 756482048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038928786359077234, + "loss": 3.0294, + "theoretical_loss": 3.749962637223984, + "tokens_seen": 756547584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892778335005015, + "loss": 2.7881, + "theoretical_loss": 3.74993011140879, + "tokens_seen": 756613120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892678034102307, + "loss": 2.6948, + "theoretical_loss": 3.7498975891995503, + "tokens_seen": 756678656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892577733199599, + "loss": 2.7056, + "theoretical_loss": 3.7498650705955523, + "tokens_seen": 756744192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038924774322968906, + "loss": 2.6511, + "theoretical_loss": 3.7498325555960843, + "tokens_seen": 756809728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038923771313941825, + "loss": 2.7346, + "theoretical_loss": 3.749800044200434, + "tokens_seen": 756875264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 426245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.891552448272705, + "objective/train/theoretical_loss": 3.749767536407891, + "objective/train/tokens_used": 777400800, + "theoretical_loss": 3.749767536407891, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892276830491475, + "loss": 2.7255, + "theoretical_loss": 3.749767536407891, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892176529588766, + "loss": 2.753, + "theoretical_loss": 3.7497350322177434, + "tokens_seen": 757006336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038920762286860584, + "loss": 2.7117, + "theoretical_loss": 3.7497025316292802, + "tokens_seen": 757071872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038919759277833497, + "loss": 2.6464, + "theoretical_loss": 3.749670034641791, + "tokens_seen": 757137408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891875626880642, + "loss": 2.8426, + "theoretical_loss": 3.7496375412545646, + "tokens_seen": 757202944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891775325977934, + "loss": 2.6978, + "theoretical_loss": 3.749605051466891, + "tokens_seen": 757268480 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038916750250752257, + "loss": 2.936, + "theoretical_loss": 3.74957256527806, + "tokens_seen": 757334016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038915747241725175, + "loss": 2.8052, + "theoretical_loss": 3.7495400826873615, + "tokens_seen": 757399552 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389147442326981, + "loss": 2.7019, + "theoretical_loss": 3.7495076036940853, + "tokens_seen": 757465088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891374122367101, + "loss": 2.8646, + "theoretical_loss": 3.749475128297523, + "tokens_seen": 757530624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038912738214643935, + "loss": 2.6208, + "theoretical_loss": 3.7494426564969636, + "tokens_seen": 757596160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891173520561685, + "loss": 2.8859, + "theoretical_loss": 3.749410188291699, + "tokens_seen": 757661696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891073219658977, + "loss": 2.7943, + "theoretical_loss": 3.74937772368102, + "tokens_seen": 757727232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890972918756269, + "loss": 2.7624, + "theoretical_loss": 3.7493452626642174, + "tokens_seen": 757792768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038908726178535607, + "loss": 2.7132, + "theoretical_loss": 3.7493128052405824, + "tokens_seen": 757858304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038907723169508525, + "loss": 2.7807, + "theoretical_loss": 3.7492803514094075, + "tokens_seen": 757923840 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038906720160481443, + "loss": 2.6621, + "theoretical_loss": 3.7492479011699835, + "tokens_seen": 757989376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890571715145436, + "loss": 2.8097, + "theoretical_loss": 3.749215454521604, + "tokens_seen": 758054912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038904714142427285, + "loss": 2.7598, + "theoretical_loss": 3.7491830114635594, + "tokens_seen": 758120448 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389037111334002, + "loss": 2.876, + "theoretical_loss": 3.749150571995143, + "tokens_seen": 758185984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890270812437312, + "loss": 2.8083, + "theoretical_loss": 3.749118136115647, + "tokens_seen": 758251520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890170511534604, + "loss": 2.6312, + "theoretical_loss": 3.749085703824364, + "tokens_seen": 758317056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890070210631896, + "loss": 2.7658, + "theoretical_loss": 3.7490532751205885, + "tokens_seen": 758382592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038899699097291876, + "loss": 2.7504, + "theoretical_loss": 3.749020850003612, + "tokens_seen": 758448128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038898696088264794, + "loss": 2.6115, + "theoretical_loss": 3.7489884284727286, + "tokens_seen": 758513664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 426718, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.676023244857788, + "objective/train/theoretical_loss": 3.748956010527232, + "objective/train/tokens_used": 779039200, + "theoretical_loss": 3.748956010527232, + "tokens_seen": 758579200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889769307923771, + "loss": 2.7646, + "theoretical_loss": 3.748956010527232, + "tokens_seen": 758579200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038896690070210635, + "loss": 2.8833, + "theoretical_loss": 3.7489235961664154, + "tokens_seen": 758644736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889568706118355, + "loss": 2.7432, + "theoretical_loss": 3.748891185389574, + "tokens_seen": 758710272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889468405215647, + "loss": 2.5897, + "theoretical_loss": 3.7488587781960008, + "tokens_seen": 758775808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038893681043129384, + "loss": 2.9097, + "theoretical_loss": 3.748826374584991, + "tokens_seen": 758841344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889267803410231, + "loss": 2.9009, + "theoretical_loss": 3.748793974555838, + "tokens_seen": 758906880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038891675025075226, + "loss": 2.6654, + "theoretical_loss": 3.748761578107838, + "tokens_seen": 758972416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038890672016048144, + "loss": 2.9037, + "theoretical_loss": 3.7487291852402853, + "tokens_seen": 759037952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888966900702106, + "loss": 2.5612, + "theoretical_loss": 3.7486967959524753, + "tokens_seen": 759103488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888866599799398, + "loss": 2.5881, + "theoretical_loss": 3.7486644102437037, + "tokens_seen": 759169024 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038887662988966904, + "loss": 2.9656, + "theoretical_loss": 3.7486320281132657, + "tokens_seen": 759234560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888665997993982, + "loss": 2.801, + "theoretical_loss": 3.748599649560457, + "tokens_seen": 759300096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888565697091274, + "loss": 2.7532, + "theoretical_loss": 3.7485672745845733, + "tokens_seen": 759365632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888465396188566, + "loss": 2.4411, + "theoretical_loss": 3.748534903184912, + "tokens_seen": 759431168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888365095285858, + "loss": 2.7694, + "theoretical_loss": 3.7485025353607684, + "tokens_seen": 759496704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038882647943831494, + "loss": 2.6058, + "theoretical_loss": 3.7484701711114394, + "tokens_seen": 759562240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888164493480442, + "loss": 2.8315, + "theoretical_loss": 3.748437810436222, + "tokens_seen": 759627776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888064192577733, + "loss": 2.9115, + "theoretical_loss": 3.748405453334413, + "tokens_seen": 759693312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038879638916750254, + "loss": 2.6568, + "theoretical_loss": 3.7483730998053097, + "tokens_seen": 759758848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887863590772317, + "loss": 2.7921, + "theoretical_loss": 3.748340749848209, + "tokens_seen": 759824384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887763289869609, + "loss": 2.8153, + "theoretical_loss": 3.74830840346241, + "tokens_seen": 759889920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887662988966901, + "loss": 2.6874, + "theoretical_loss": 3.7482760606472088, + "tokens_seen": 759955456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038875626880641927, + "loss": 2.949, + "theoretical_loss": 3.748243721401904, + "tokens_seen": 760020992 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038874623871614845, + "loss": 2.771, + "theoretical_loss": 3.748211385725793, + "tokens_seen": 760086528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887362086258777, + "loss": 2.7016, + "theoretical_loss": 3.748179053618176, + "tokens_seen": 760152064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 427515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.012765645980835, + "objective/train/theoretical_loss": 3.7481467250783504, + "objective/train/tokens_used": 780677600, + "theoretical_loss": 3.7481467250783504, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887261785356068, + "loss": 3.079, + "theoretical_loss": 3.7481467250783504, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038871614844533604, + "loss": 2.777, + "theoretical_loss": 3.748114400105615, + "tokens_seen": 760283136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038870611835506517, + "loss": 2.8811, + "theoretical_loss": 3.748082078699269, + "tokens_seen": 760348672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886960882647944, + "loss": 2.8731, + "theoretical_loss": 3.7480497608586116, + "tokens_seen": 760414208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886860581745236, + "loss": 2.8526, + "theoretical_loss": 3.748017446582942, + "tokens_seen": 760479744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038867602808425277, + "loss": 2.9608, + "theoretical_loss": 3.74798513587156, + "tokens_seen": 760545280 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038866599799398195, + "loss": 2.7168, + "theoretical_loss": 3.747952828723765, + "tokens_seen": 760610816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886559679037112, + "loss": 2.7147, + "theoretical_loss": 3.7479205251388574, + "tokens_seen": 760676352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886459378134403, + "loss": 2.8606, + "theoretical_loss": 3.747888225116137, + "tokens_seen": 760741888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038863590772316955, + "loss": 2.9467, + "theoretical_loss": 3.7478559286549045, + "tokens_seen": 760807424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886258776328987, + "loss": 2.7865, + "theoretical_loss": 3.74782363575446, + "tokens_seen": 760872960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886158475426279, + "loss": 2.7565, + "theoretical_loss": 3.747791346414105, + "tokens_seen": 760938496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886058174523571, + "loss": 2.7405, + "theoretical_loss": 3.74775906063314, + "tokens_seen": 761004032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038859578736208627, + "loss": 2.7645, + "theoretical_loss": 3.747726778410866, + "tokens_seen": 761069568 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038858575727181545, + "loss": 2.9492, + "theoretical_loss": 3.7476944997465846, + "tokens_seen": 761135104 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038857572718154463, + "loss": 2.8445, + "theoretical_loss": 3.747662224639597, + "tokens_seen": 761200640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885656970912738, + "loss": 2.7768, + "theoretical_loss": 3.7476299530892057, + "tokens_seen": 761266176 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038855566700100305, + "loss": 3.0213, + "theoretical_loss": 3.7475976850947124, + "tokens_seen": 761331712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885456369107322, + "loss": 2.9531, + "theoretical_loss": 3.7475654206554188, + "tokens_seen": 761397248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885356068204614, + "loss": 2.9068, + "theoretical_loss": 3.7475331597706267, + "tokens_seen": 761462784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885255767301906, + "loss": 2.7472, + "theoretical_loss": 3.74750090243964, + "tokens_seen": 761528320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885155466399198, + "loss": 2.8358, + "theoretical_loss": 3.747468648661761, + "tokens_seen": 761593856 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038850551654964896, + "loss": 3.0352, + "theoretical_loss": 3.7474363984362924, + "tokens_seen": 761659392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038849548645937814, + "loss": 2.7928, + "theoretical_loss": 3.7474041517625376, + "tokens_seen": 761724928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884854563691073, + "loss": 2.9632, + "theoretical_loss": 3.7473719086397996, + "tokens_seen": 761790464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 428729, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0259807109832764, + "objective/train/theoretical_loss": 3.747339669067382, + "objective/train/tokens_used": 782316000, + "theoretical_loss": 3.747339669067382, + "tokens_seen": 761856000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038847542627883655, + "loss": 3.03, + "theoretical_loss": 3.747339669067382, + "tokens_seen": 761856000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884653961885657, + "loss": 2.7308, + "theoretical_loss": 3.747307433044589, + "tokens_seen": 761921536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884553660982949, + "loss": 2.7257, + "theoretical_loss": 3.7472752005707233, + "tokens_seen": 761987072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038844533600802404, + "loss": 2.8627, + "theoretical_loss": 3.7472429716450897, + "tokens_seen": 762052608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884353059177533, + "loss": 2.8747, + "theoretical_loss": 3.747210746266993, + "tokens_seen": 762118144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038842527582748246, + "loss": 2.8288, + "theoretical_loss": 3.747178524435738, + "tokens_seen": 762183680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038841524573721164, + "loss": 2.7417, + "theoretical_loss": 3.747146306150628, + "tokens_seen": 762249216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884052156469408, + "loss": 2.8478, + "theoretical_loss": 3.7471140914109684, + "tokens_seen": 762314752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038839518555667, + "loss": 2.6023, + "theoretical_loss": 3.7470818802160646, + "tokens_seen": 762380288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883851554663992, + "loss": 2.88, + "theoretical_loss": 3.747049672565222, + "tokens_seen": 762445824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883751253761284, + "loss": 2.9135, + "theoretical_loss": 3.747017468457746, + "tokens_seen": 762511360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038836509528585755, + "loss": 3.1032, + "theoretical_loss": 3.7469852678929416, + "tokens_seen": 762576896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883550651955868, + "loss": 2.8413, + "theoretical_loss": 3.7469530708701155, + "tokens_seen": 762642432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038834503510531596, + "loss": 2.6678, + "theoretical_loss": 3.7469208773885736, + "tokens_seen": 762707968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038833500501504514, + "loss": 2.764, + "theoretical_loss": 3.7468886874476217, + "tokens_seen": 762773504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883249749247743, + "loss": 2.7965, + "theoretical_loss": 3.746856501046567, + "tokens_seen": 762839040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883149448345035, + "loss": 2.6913, + "theoretical_loss": 3.746824318184715, + "tokens_seen": 762904576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883049147442327, + "loss": 2.9067, + "theoretical_loss": 3.7467921388613745, + "tokens_seen": 762970112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882948846539619, + "loss": 2.7015, + "theoretical_loss": 3.746759963075851, + "tokens_seen": 763035648 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038828485456369105, + "loss": 2.962, + "theoretical_loss": 3.7467277908274514, + "tokens_seen": 763101184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882748244734203, + "loss": 3.0969, + "theoretical_loss": 3.7466956221154843, + "tokens_seen": 763166720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882647943831494, + "loss": 2.8899, + "theoretical_loss": 3.746663456939257, + "tokens_seen": 763232256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038825476429287865, + "loss": 3.1046, + "theoretical_loss": 3.7466312952980774, + "tokens_seen": 763297792 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038824473420260783, + "loss": 2.7427, + "theoretical_loss": 3.746599137191253, + "tokens_seen": 763363328 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388234704112337, + "loss": 2.9909, + "theoretical_loss": 3.7465669826180923, + "tokens_seen": 763428864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 429385, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7105305194854736, + "objective/train/theoretical_loss": 3.746534831577904, + "objective/train/tokens_used": 783954400, + "theoretical_loss": 3.746534831577904, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882246740220662, + "loss": 2.7427, + "theoretical_loss": 3.746534831577904, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038821464393179537, + "loss": 2.726, + "theoretical_loss": 3.7465026840699958, + "tokens_seen": 763559936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038820461384152455, + "loss": 2.7427, + "theoretical_loss": 3.7464705400936777, + "tokens_seen": 763625472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881945837512538, + "loss": 2.7322, + "theoretical_loss": 3.746438399648258, + "tokens_seen": 763691008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881845536609829, + "loss": 2.996, + "theoretical_loss": 3.7464062627330463, + "tokens_seen": 763756544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038817452357071215, + "loss": 2.5489, + "theoretical_loss": 3.7463741293473514, + "tokens_seen": 763822080 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038816449348044133, + "loss": 2.7436, + "theoretical_loss": 3.7463419994904834, + "tokens_seen": 763887616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881544633901705, + "loss": 2.758, + "theoretical_loss": 3.746309873161752, + "tokens_seen": 763953152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881444332998997, + "loss": 2.8826, + "theoretical_loss": 3.7462777503604667, + "tokens_seen": 764018688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881344032096289, + "loss": 2.7736, + "theoretical_loss": 3.746245631085938, + "tokens_seen": 764084224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881243731193581, + "loss": 2.6549, + "theoretical_loss": 3.7462135153374767, + "tokens_seen": 764149760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881143430290873, + "loss": 2.7209, + "theoretical_loss": 3.7461814031143925, + "tokens_seen": 764215296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038810431293881647, + "loss": 2.7706, + "theoretical_loss": 3.746149294415997, + "tokens_seen": 764280832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038809428284854565, + "loss": 2.7119, + "theoretical_loss": 3.7461171892416, + "tokens_seen": 764346368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038808425275827483, + "loss": 2.4525, + "theoretical_loss": 3.746085087590514, + "tokens_seen": 764411904 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388074222668004, + "loss": 2.7598, + "theoretical_loss": 3.7460529894620493, + "tokens_seen": 764477440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038806419257773325, + "loss": 2.6697, + "theoretical_loss": 3.7460208948555183, + "tokens_seen": 764542976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880541624874624, + "loss": 2.7454, + "theoretical_loss": 3.7459888037702314, + "tokens_seen": 764608512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880441323971916, + "loss": 2.8053, + "theoretical_loss": 3.7459567162055016, + "tokens_seen": 764674048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880341023069208, + "loss": 2.5966, + "theoretical_loss": 3.7459246321606408, + "tokens_seen": 764739584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038802407221665, + "loss": 2.8628, + "theoretical_loss": 3.7458925516349613, + "tokens_seen": 764805120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038801404212637916, + "loss": 2.7437, + "theoretical_loss": 3.745860474627775, + "tokens_seen": 764870656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038800401203610834, + "loss": 2.7495, + "theoretical_loss": 3.7458284011383953, + "tokens_seen": 764936192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879939819458375, + "loss": 2.5588, + "theoretical_loss": 3.745796331166135, + "tokens_seen": 765001728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038798395185556675, + "loss": 2.6211, + "theoretical_loss": 3.745764264710307, + "tokens_seen": 765067264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 430652, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8052632808685303, + "objective/train/theoretical_loss": 3.7457322017702244, + "objective/train/tokens_used": 785592800, + "theoretical_loss": 3.7457322017702244, + "tokens_seen": 765132800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879739217652959, + "loss": 2.8717, + "theoretical_loss": 3.7457322017702244, + "tokens_seen": 765132800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879638916750251, + "loss": 2.8154, + "theoretical_loss": 3.7457001423452008, + "tokens_seen": 765198336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038795386158475424, + "loss": 2.8259, + "theoretical_loss": 3.7456680864345495, + "tokens_seen": 765263872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879438314944835, + "loss": 2.955, + "theoretical_loss": 3.745636034037585, + "tokens_seen": 765329408 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038793380140421266, + "loss": 2.7045, + "theoretical_loss": 3.745603985153621, + "tokens_seen": 765394944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038792377131394184, + "loss": 2.7434, + "theoretical_loss": 3.745571939781972, + "tokens_seen": 765460480 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387913741223671, + "loss": 2.6924, + "theoretical_loss": 3.7455398979219514, + "tokens_seen": 765526016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879037111334002, + "loss": 2.7036, + "theoretical_loss": 3.745507859572875, + "tokens_seen": 765591552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878936810431294, + "loss": 2.8034, + "theoretical_loss": 3.745475824734057, + "tokens_seen": 765657088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878836509528586, + "loss": 2.9501, + "theoretical_loss": 3.745443793404813, + "tokens_seen": 765722624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038787362086258775, + "loss": 2.675, + "theoretical_loss": 3.745411765584457, + "tokens_seen": 765788160 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387863590772317, + "loss": 2.8542, + "theoretical_loss": 3.7453797412723056, + "tokens_seen": 765853696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038785356068204616, + "loss": 2.7184, + "theoretical_loss": 3.7453477204676737, + "tokens_seen": 765919232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038784353059177534, + "loss": 2.8863, + "theoretical_loss": 3.7453157031698776, + "tokens_seen": 765984768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878335005015045, + "loss": 2.6537, + "theoretical_loss": 3.745283689378232, + "tokens_seen": 766050304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878234704112337, + "loss": 2.8882, + "theoretical_loss": 3.7452516790920543, + "tokens_seen": 766115840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878134403209629, + "loss": 2.7967, + "theoretical_loss": 3.745219672310661, + "tokens_seen": 766181376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878034102306921, + "loss": 2.8745, + "theoretical_loss": 3.745187669033368, + "tokens_seen": 766246912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038779338014042125, + "loss": 2.7758, + "theoretical_loss": 3.745155669259491, + "tokens_seen": 766312448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877833500501505, + "loss": 2.8668, + "theoretical_loss": 3.745123672988349, + "tokens_seen": 766377984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877733199598796, + "loss": 2.8476, + "theoretical_loss": 3.745091680219258, + "tokens_seen": 766443520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038776328986960885, + "loss": 2.8759, + "theoretical_loss": 3.7450596909515355, + "tokens_seen": 766509056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038775325977933803, + "loss": 3.0193, + "theoretical_loss": 3.7450277051844987, + "tokens_seen": 766574592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877432296890672, + "loss": 2.7882, + "theoretical_loss": 3.7449957229174653, + "tokens_seen": 766640128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877331995987964, + "loss": 2.9495, + "theoretical_loss": 3.7449637441497536, + "tokens_seen": 766705664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 431055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6919522285461426, + "objective/train/theoretical_loss": 3.744931768880681, + "objective/train/tokens_used": 787231200, + "theoretical_loss": 3.744931768880681, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038772316950852557, + "loss": 2.92, + "theoretical_loss": 3.744931768880681, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038771313941825475, + "loss": 2.7832, + "theoretical_loss": 3.744899797109567, + "tokens_seen": 766836736 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387703109327984, + "loss": 2.8811, + "theoretical_loss": 3.7448678288357287, + "tokens_seen": 766902272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876930792377131, + "loss": 2.921, + "theoretical_loss": 3.7448358640584853, + "tokens_seen": 766967808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038768304914744235, + "loss": 2.9364, + "theoretical_loss": 3.7448039027771554, + "tokens_seen": 767033344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038767301905717153, + "loss": 2.5438, + "theoretical_loss": 3.744771944991058, + "tokens_seen": 767098880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876629889669007, + "loss": 2.7965, + "theoretical_loss": 3.7447399906995127, + "tokens_seen": 767164416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876529588766299, + "loss": 2.8438, + "theoretical_loss": 3.7447080399018393, + "tokens_seen": 767229952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876429287863591, + "loss": 2.9344, + "theoretical_loss": 3.7446760925973557, + "tokens_seen": 767295488 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038763289869608826, + "loss": 2.7249, + "theoretical_loss": 3.744644148785383, + "tokens_seen": 767361024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876228686058175, + "loss": 2.8412, + "theoretical_loss": 3.7446122084652416, + "tokens_seen": 767426560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876128385155466, + "loss": 2.777, + "theoretical_loss": 3.7445802716362504, + "tokens_seen": 767492096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038760280842527585, + "loss": 2.8745, + "theoretical_loss": 3.7445483382977303, + "tokens_seen": 767557632 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387592778335005, + "loss": 2.7448, + "theoretical_loss": 3.744516408449002, + "tokens_seen": 767623168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875827482447342, + "loss": 2.9832, + "theoretical_loss": 3.744484482089386, + "tokens_seen": 767688704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875727181544634, + "loss": 2.8086, + "theoretical_loss": 3.744452559218203, + "tokens_seen": 767754240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875626880641926, + "loss": 2.6926, + "theoretical_loss": 3.7444206398347752, + "tokens_seen": 767819776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038755265797392176, + "loss": 2.8013, + "theoretical_loss": 3.744388723938423, + "tokens_seen": 767885312 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387542627883651, + "loss": 2.776, + "theoretical_loss": 3.7443568115284673, + "tokens_seen": 767950848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875325977933801, + "loss": 2.659, + "theoretical_loss": 3.744324902604231, + "tokens_seen": 768016384 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038752256770310936, + "loss": 2.7347, + "theoretical_loss": 3.7442929971650356, + "tokens_seen": 768081920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875125376128385, + "loss": 2.7669, + "theoretical_loss": 3.744261095210203, + "tokens_seen": 768147456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875025075225677, + "loss": 2.8019, + "theoretical_loss": 3.744229196739055, + "tokens_seen": 768212992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874924774322969, + "loss": 2.8041, + "theoretical_loss": 3.7441973017509147, + "tokens_seen": 768278528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874824473420261, + "loss": 2.8703, + "theoretical_loss": 3.7441654102451047, + "tokens_seen": 768344064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 432209, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0975613594055176, + "objective/train/theoretical_loss": 3.744133522220947, + "objective/train/tokens_used": 788869600, + "theoretical_loss": 3.744133522220947, + "tokens_seen": 768409600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038747241725175526, + "loss": 2.8819, + "theoretical_loss": 3.744133522220947, + "tokens_seen": 768409600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038746238716148444, + "loss": 2.7959, + "theoretical_loss": 3.7441016376777663, + "tokens_seen": 768475136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874523570712136, + "loss": 2.9047, + "theoretical_loss": 3.744069756614884, + "tokens_seen": 768540672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038744232698094286, + "loss": 2.8923, + "theoretical_loss": 3.744037879031625, + "tokens_seen": 768606208 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387432296890672, + "loss": 2.9207, + "theoretical_loss": 3.7440060049273116, + "tokens_seen": 768671744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874222668004012, + "loss": 2.703, + "theoretical_loss": 3.7439741343012676, + "tokens_seen": 768737280 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038741223671013035, + "loss": 2.7158, + "theoretical_loss": 3.7439422671528177, + "tokens_seen": 768802816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874022066198596, + "loss": 2.7988, + "theoretical_loss": 3.7439104034812862, + "tokens_seen": 768868352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038739217652958876, + "loss": 2.9013, + "theoretical_loss": 3.743878543285997, + "tokens_seen": 768933888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038738214643931795, + "loss": 2.932, + "theoretical_loss": 3.7438466865662745, + "tokens_seen": 768999424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873721163490472, + "loss": 2.9536, + "theoretical_loss": 3.7438148333214434, + "tokens_seen": 769064960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038736208625877636, + "loss": 2.6854, + "theoretical_loss": 3.7437829835508287, + "tokens_seen": 769130496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038735205616850554, + "loss": 2.749, + "theoretical_loss": 3.743751137253756, + "tokens_seen": 769196032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873420260782347, + "loss": 2.4222, + "theoretical_loss": 3.7437192944295496, + "tokens_seen": 769261568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873319959879639, + "loss": 2.9377, + "theoretical_loss": 3.743687455077536, + "tokens_seen": 769327104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873219658976931, + "loss": 2.5288, + "theoretical_loss": 3.74365561919704, + "tokens_seen": 769392640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873119358074223, + "loss": 2.9269, + "theoretical_loss": 3.7436237867873876, + "tokens_seen": 769458176 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038730190571715145, + "loss": 3.2117, + "theoretical_loss": 3.743591957847905, + "tokens_seen": 769523712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872918756268807, + "loss": 2.8716, + "theoretical_loss": 3.7435601323779184, + "tokens_seen": 769589248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872818455366098, + "loss": 2.9267, + "theoretical_loss": 3.743528310376754, + "tokens_seen": 769654784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038727181544633905, + "loss": 2.8562, + "theoretical_loss": 3.7434964918437394, + "tokens_seen": 769720320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038726178535606823, + "loss": 2.617, + "theoretical_loss": 3.7434646767782, + "tokens_seen": 769785856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872517552657974, + "loss": 2.8305, + "theoretical_loss": 3.7434328651794635, + "tokens_seen": 769851392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872417251755266, + "loss": 2.8065, + "theoretical_loss": 3.743401057046857, + "tokens_seen": 769916928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038723169508525577, + "loss": 2.7927, + "theoretical_loss": 3.7433692523797077, + "tokens_seen": 769982464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 432880, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1428420543670654, + "objective/train/theoretical_loss": 3.743337451177343, + "objective/train/tokens_used": 790508000, + "theoretical_loss": 3.743337451177343, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038722166499498495, + "loss": 3.0053, + "theoretical_loss": 3.743337451177343, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872116349047142, + "loss": 2.8568, + "theoretical_loss": 3.743305653439091, + "tokens_seen": 770113536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872016048144433, + "loss": 2.941, + "theoretical_loss": 3.7432738591642796, + "tokens_seen": 770179072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038719157472417255, + "loss": 2.8203, + "theoretical_loss": 3.7432420683522363, + "tokens_seen": 770244608 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038718154463390173, + "loss": 2.8672, + "theoretical_loss": 3.7432102810022903, + "tokens_seen": 770310144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871715145436309, + "loss": 2.8655, + "theoretical_loss": 3.743178497113769, + "tokens_seen": 770375680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871614844533601, + "loss": 2.7875, + "theoretical_loss": 3.7431467166860015, + "tokens_seen": 770441216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871514543630893, + "loss": 2.7909, + "theoretical_loss": 3.7431149397183177, + "tokens_seen": 770506752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038714142427281846, + "loss": 2.8959, + "theoretical_loss": 3.743083166210045, + "tokens_seen": 770572288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871313941825477, + "loss": 2.7932, + "theoretical_loss": 3.7430513961605136, + "tokens_seen": 770637824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871213640922768, + "loss": 2.814, + "theoretical_loss": 3.7430196295690523, + "tokens_seen": 770703360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038711133400200605, + "loss": 3.0201, + "theoretical_loss": 3.7429878664349916, + "tokens_seen": 770768896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871013039117352, + "loss": 2.8408, + "theoretical_loss": 3.74295610675766, + "tokens_seen": 770834432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870912738214644, + "loss": 2.6841, + "theoretical_loss": 3.7429243505363887, + "tokens_seen": 770899968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870812437311936, + "loss": 2.7229, + "theoretical_loss": 3.742892597770507, + "tokens_seen": 770965504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870712136409228, + "loss": 2.8088, + "theoretical_loss": 3.7428608484593457, + "tokens_seen": 771031040 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038706118355065196, + "loss": 2.6387, + "theoretical_loss": 3.7428291026022356, + "tokens_seen": 771096576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870511534603812, + "loss": 2.7468, + "theoretical_loss": 3.742797360198507, + "tokens_seen": 771162112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870411233701103, + "loss": 2.7474, + "theoretical_loss": 3.7427656212474902, + "tokens_seen": 771227648 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038703109327983956, + "loss": 2.8616, + "theoretical_loss": 3.7427338857485175, + "tokens_seen": 771293184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870210631895687, + "loss": 2.6703, + "theoretical_loss": 3.742702153700919, + "tokens_seen": 771358720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870110330992979, + "loss": 2.7043, + "theoretical_loss": 3.742670425104027, + "tokens_seen": 771424256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870010030090271, + "loss": 2.797, + "theoretical_loss": 3.7426386999571735, + "tokens_seen": 771489792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869909729187563, + "loss": 2.8079, + "theoretical_loss": 3.7426069782596887, + "tokens_seen": 771555328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038698094282848546, + "loss": 2.8836, + "theoretical_loss": 3.7425752600109066, + "tokens_seen": 771620864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 434052, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.293273448944092, + "objective/train/theoretical_loss": 3.742543545210158, + "objective/train/tokens_used": 792146400, + "theoretical_loss": 3.742543545210158, + "tokens_seen": 771686400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038697091273821464, + "loss": 2.6631, + "theoretical_loss": 3.742543545210158, + "tokens_seen": 771686400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869608826479438, + "loss": 2.892, + "theoretical_loss": 3.7425118338567755, + "tokens_seen": 771751936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038695085255767306, + "loss": 2.8685, + "theoretical_loss": 3.7424801259500926, + "tokens_seen": 771817472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869408224674022, + "loss": 2.8693, + "theoretical_loss": 3.7424484214894407, + "tokens_seen": 771883008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869307923771314, + "loss": 2.8568, + "theoretical_loss": 3.7424167204741536, + "tokens_seen": 771948544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038692076228686055, + "loss": 2.937, + "theoretical_loss": 3.7423850229035645, + "tokens_seen": 772014080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869107321965898, + "loss": 2.6878, + "theoretical_loss": 3.742353328777006, + "tokens_seen": 772079616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038690070210631896, + "loss": 2.7514, + "theoretical_loss": 3.742321638093813, + "tokens_seen": 772145152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038689067201604815, + "loss": 2.8053, + "theoretical_loss": 3.7422899508533174, + "tokens_seen": 772210688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038688064192577733, + "loss": 2.6689, + "theoretical_loss": 3.7422582670548543, + "tokens_seen": 772276224 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038687061183550656, + "loss": 2.6903, + "theoretical_loss": 3.7422265866977575, + "tokens_seen": 772341760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868605817452357, + "loss": 2.7236, + "theoretical_loss": 3.742194909781361, + "tokens_seen": 772407296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868505516549649, + "loss": 2.9353, + "theoretical_loss": 3.7421632363049993, + "tokens_seen": 772472832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038684052156469405, + "loss": 2.823, + "theoretical_loss": 3.7421315662680072, + "tokens_seen": 772538368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868304914744233, + "loss": 2.7551, + "theoretical_loss": 3.7420998996697197, + "tokens_seen": 772603904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038682046138415247, + "loss": 2.9628, + "theoretical_loss": 3.7420682365094713, + "tokens_seen": 772669440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038681043129388165, + "loss": 2.7756, + "theoretical_loss": 3.7420365767865977, + "tokens_seen": 772734976 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038680040120361083, + "loss": 2.9, + "theoretical_loss": 3.7420049205004338, + "tokens_seen": 772800512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038679037111334, + "loss": 2.8719, + "theoretical_loss": 3.741973267650315, + "tokens_seen": 772866048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867803410230692, + "loss": 2.9457, + "theoretical_loss": 3.741941618235577, + "tokens_seen": 772931584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038677031093279843, + "loss": 2.8025, + "theoretical_loss": 3.7419099722555567, + "tokens_seen": 772997120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038676028084252755, + "loss": 2.6723, + "theoretical_loss": 3.7418783297095892, + "tokens_seen": 773062656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867502507522568, + "loss": 2.8905, + "theoretical_loss": 3.741846690597012, + "tokens_seen": 773128192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867402206619859, + "loss": 2.7707, + "theoretical_loss": 3.7418150549171596, + "tokens_seen": 773193728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038673019057171515, + "loss": 2.873, + "theoretical_loss": 3.74178342266937, + "tokens_seen": 773259264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 434666, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.016434907913208, + "objective/train/theoretical_loss": 3.74175179385298, + "objective/train/tokens_used": 793784800, + "theoretical_loss": 3.74175179385298, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038672016048144433, + "loss": 2.8806, + "theoretical_loss": 3.74175179385298, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867101303911735, + "loss": 2.7735, + "theoretical_loss": 3.7417201684673262, + "tokens_seen": 773390336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867001003009027, + "loss": 2.8635, + "theoretical_loss": 3.741688546511746, + "tokens_seen": 773455872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038669007021063193, + "loss": 2.7011, + "theoretical_loss": 3.7416569279855767, + "tokens_seen": 773521408 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038668004012036106, + "loss": 2.6612, + "theoretical_loss": 3.7416253128881563, + "tokens_seen": 773586944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866700100300903, + "loss": 3.141, + "theoretical_loss": 3.7415937012188216, + "tokens_seen": 773652480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866599799398194, + "loss": 2.8049, + "theoretical_loss": 3.7415620929769116, + "tokens_seen": 773718016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038664994984954866, + "loss": 2.7794, + "theoretical_loss": 3.7415304881617644, + "tokens_seen": 773783552 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038663991975927784, + "loss": 2.9484, + "theoretical_loss": 3.7414988867727166, + "tokens_seen": 773849088 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386629889669007, + "loss": 2.8271, + "theoretical_loss": 3.741467288809109, + "tokens_seen": 773914624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038661985957873625, + "loss": 2.9336, + "theoretical_loss": 3.7414356942702787, + "tokens_seen": 773980160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866098294884654, + "loss": 2.8063, + "theoretical_loss": 3.7414041031555656, + "tokens_seen": 774045696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865997993981946, + "loss": 2.7585, + "theoretical_loss": 3.7413725154643083, + "tokens_seen": 774111232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865897693079238, + "loss": 2.7239, + "theoretical_loss": 3.741340931195846, + "tokens_seen": 774176768 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386579739217653, + "loss": 2.6134, + "theoretical_loss": 3.7413093503495176, + "tokens_seen": 774242304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038656970912738216, + "loss": 2.9152, + "theoretical_loss": 3.741277772924663, + "tokens_seen": 774307840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865596790371114, + "loss": 2.7841, + "theoretical_loss": 3.7412461989206225, + "tokens_seen": 774373376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865496489468405, + "loss": 2.7092, + "theoretical_loss": 3.741214628336736, + "tokens_seen": 774438912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038653961885656976, + "loss": 2.724, + "theoretical_loss": 3.741183061172343, + "tokens_seen": 774504448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865295887662989, + "loss": 2.8625, + "theoretical_loss": 3.7411514974267845, + "tokens_seen": 774569984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865195586760281, + "loss": 3.0894, + "theoretical_loss": 3.7411199370994, + "tokens_seen": 774635520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865095285857573, + "loss": 2.8484, + "theoretical_loss": 3.741088380189532, + "tokens_seen": 774701056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864994984954865, + "loss": 2.7995, + "theoretical_loss": 3.7410568266965196, + "tokens_seen": 774766592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038648946840521566, + "loss": 2.8275, + "theoretical_loss": 3.7410252766197045, + "tokens_seen": 774832128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038647943831494484, + "loss": 2.7769, + "theoretical_loss": 3.740993729958428, + "tokens_seen": 774897664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 435763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.750361680984497, + "objective/train/theoretical_loss": 3.7409621867120313, + "objective/train/tokens_used": 795423200, + "theoretical_loss": 3.7409621867120313, + "tokens_seen": 774963200 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386469408224674, + "loss": 2.6144, + "theoretical_loss": 3.7409621867120313, + "tokens_seen": 774963200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038645937813440326, + "loss": 2.5263, + "theoretical_loss": 3.7409306468798564, + "tokens_seen": 775028736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864493480441324, + "loss": 2.7634, + "theoretical_loss": 3.7408991104612452, + "tokens_seen": 775094272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864393179538616, + "loss": 2.7988, + "theoretical_loss": 3.740867577455539, + "tokens_seen": 775159808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038642928786359075, + "loss": 2.9711, + "theoretical_loss": 3.7408360478620803, + "tokens_seen": 775225344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038641925777332, + "loss": 2.7774, + "theoretical_loss": 3.7408045216802117, + "tokens_seen": 775290880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038640922768304917, + "loss": 2.8231, + "theoretical_loss": 3.740772998909275, + "tokens_seen": 775356416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038639919759277835, + "loss": 2.9609, + "theoretical_loss": 3.740741479548613, + "tokens_seen": 775421952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038638916750250753, + "loss": 2.8501, + "theoretical_loss": 3.74070996359757, + "tokens_seen": 775487488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038637913741223676, + "loss": 2.7855, + "theoretical_loss": 3.7406784510554876, + "tokens_seen": 775553024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863691073219659, + "loss": 2.5942, + "theoretical_loss": 3.7406469419217094, + "tokens_seen": 775618560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863590772316951, + "loss": 2.7587, + "theoretical_loss": 3.7406154361955792, + "tokens_seen": 775684096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038634904714142425, + "loss": 2.4622, + "theoretical_loss": 3.7405839338764397, + "tokens_seen": 775749632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863390170511535, + "loss": 2.7932, + "theoretical_loss": 3.740552434963636, + "tokens_seen": 775815168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038632898696088267, + "loss": 2.8275, + "theoretical_loss": 3.740520939456511, + "tokens_seen": 775880704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038631895687061185, + "loss": 2.9642, + "theoretical_loss": 3.740489447354409, + "tokens_seen": 775946240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038630892678034103, + "loss": 2.7347, + "theoretical_loss": 3.7404579586566746, + "tokens_seen": 776011776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862988966900702, + "loss": 2.6956, + "theoretical_loss": 3.7404264733626524, + "tokens_seen": 776077312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862888665997994, + "loss": 3.0187, + "theoretical_loss": 3.740394991471687, + "tokens_seen": 776142848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038627883650952863, + "loss": 2.956, + "theoretical_loss": 3.740363512983123, + "tokens_seen": 776208384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038626880641925775, + "loss": 2.7489, + "theoretical_loss": 3.740332037896306, + "tokens_seen": 776273920 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386258776328987, + "loss": 2.8328, + "theoretical_loss": 3.740300566210581, + "tokens_seen": 776339456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862487462387161, + "loss": 3.0334, + "theoretical_loss": 3.7402690979252937, + "tokens_seen": 776404992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038623871614844535, + "loss": 2.8642, + "theoretical_loss": 3.740237633039789, + "tokens_seen": 776470528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038622868605817453, + "loss": 2.7713, + "theoretical_loss": 3.7402061715534134, + "tokens_seen": 776536064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 436694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.752096652984619, + "objective/train/theoretical_loss": 3.740174713465512, + "objective/train/tokens_used": 797061600, + "theoretical_loss": 3.740174713465512, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862186559679037, + "loss": 2.7836, + "theoretical_loss": 3.740174713465512, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862086258776329, + "loss": 2.7319, + "theoretical_loss": 3.7401432587754324, + "tokens_seen": 776667136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038619859578736213, + "loss": 2.9838, + "theoretical_loss": 3.7401118074825197, + "tokens_seen": 776732672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038618856569709126, + "loss": 2.8541, + "theoretical_loss": 3.7400803595861203, + "tokens_seen": 776798208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861785356068205, + "loss": 2.7572, + "theoretical_loss": 3.740048915085582, + "tokens_seen": 776863744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861685055165496, + "loss": 3.0645, + "theoretical_loss": 3.7400174739802505, + "tokens_seen": 776929280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038615847542627886, + "loss": 2.8094, + "theoretical_loss": 3.7399860362694737, + "tokens_seen": 776994816 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038614844533600804, + "loss": 2.7469, + "theoretical_loss": 3.7399546019525984, + "tokens_seen": 777060352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861384152457372, + "loss": 2.7164, + "theoretical_loss": 3.7399231710289724, + "tokens_seen": 777125888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861283851554664, + "loss": 2.7987, + "theoretical_loss": 3.7398917434979424, + "tokens_seen": 777191424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861183550651956, + "loss": 2.9797, + "theoretical_loss": 3.7398603193588578, + "tokens_seen": 777256960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038610832497492476, + "loss": 2.8151, + "theoretical_loss": 3.739828898611065, + "tokens_seen": 777322496 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386098294884654, + "loss": 2.8161, + "theoretical_loss": 3.7397974812539125, + "tokens_seen": 777388032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860882647943831, + "loss": 2.7239, + "theoretical_loss": 3.739766067286749, + "tokens_seen": 777453568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038607823470411236, + "loss": 2.7941, + "theoretical_loss": 3.739734656708923, + "tokens_seen": 777519104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860682046138415, + "loss": 2.7307, + "theoretical_loss": 3.739703249519783, + "tokens_seen": 777584640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860581745235707, + "loss": 2.7405, + "theoretical_loss": 3.7396718457186777, + "tokens_seen": 777650176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860481444332999, + "loss": 2.8985, + "theoretical_loss": 3.739640445304957, + "tokens_seen": 777715712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860381143430291, + "loss": 2.7029, + "theoretical_loss": 3.7396090482779685, + "tokens_seen": 777781248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038602808425275826, + "loss": 2.7713, + "theoretical_loss": 3.7395776546370634, + "tokens_seen": 777846784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860180541624875, + "loss": 3.0061, + "theoretical_loss": 3.7395462643815893, + "tokens_seen": 777912320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860080240722166, + "loss": 2.7808, + "theoretical_loss": 3.739514877510898, + "tokens_seen": 777977856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038599799398194586, + "loss": 2.9553, + "theoretical_loss": 3.739483494024338, + "tokens_seen": 778043392 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385987963891675, + "loss": 2.9095, + "theoretical_loss": 3.73945211392126, + "tokens_seen": 778108928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859779338014042, + "loss": 2.862, + "theoretical_loss": 3.739420737201015, + "tokens_seen": 778174464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 437315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8093600273132324, + "objective/train/theoretical_loss": 3.7393893638629514, + "objective/train/tokens_used": 798700000, + "theoretical_loss": 3.7393893638629514, + "tokens_seen": 778240000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859679037111334, + "loss": 2.887, + "theoretical_loss": 3.7393893638629514, + "tokens_seen": 778240000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859578736208626, + "loss": 2.8717, + "theoretical_loss": 3.739357993906422, + "tokens_seen": 778305536 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038594784353059177, + "loss": 2.6532, + "theoretical_loss": 3.739326627330777, + "tokens_seen": 778371072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038593781344032095, + "loss": 2.8611, + "theoretical_loss": 3.7392952641353663, + "tokens_seen": 778436608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038592778335005013, + "loss": 2.6005, + "theoretical_loss": 3.739263904319543, + "tokens_seen": 778502144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038591775325977937, + "loss": 2.7977, + "theoretical_loss": 3.739232547882657, + "tokens_seen": 778567680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859077231695085, + "loss": 2.921, + "theoretical_loss": 3.73920119482406, + "tokens_seen": 778633216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038589769307923773, + "loss": 2.6604, + "theoretical_loss": 3.7391698451431044, + "tokens_seen": 778698752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038588766298896685, + "loss": 2.8638, + "theoretical_loss": 3.7391384988391416, + "tokens_seen": 778764288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858776328986961, + "loss": 3.0366, + "theoretical_loss": 3.7391071559115243, + "tokens_seen": 778829824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858676028084253, + "loss": 2.8517, + "theoretical_loss": 3.7390758163596036, + "tokens_seen": 778895360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038585757271815445, + "loss": 2.8778, + "theoretical_loss": 3.739044480182733, + "tokens_seen": 778960896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858475426278837, + "loss": 2.49, + "theoretical_loss": 3.7390131473802652, + "tokens_seen": 779026432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038583751253761287, + "loss": 2.9749, + "theoretical_loss": 3.7389818179515517, + "tokens_seen": 779091968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038582748244734205, + "loss": 2.7421, + "theoretical_loss": 3.738950491895947, + "tokens_seen": 779157504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038581745235707123, + "loss": 2.663, + "theoretical_loss": 3.738919169212804, + "tokens_seen": 779223040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858074222668004, + "loss": 2.7339, + "theoretical_loss": 3.7388878499014746, + "tokens_seen": 779288576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857973921765296, + "loss": 2.6934, + "theoretical_loss": 3.7388565339613145, + "tokens_seen": 779354112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038578736208625883, + "loss": 2.7529, + "theoretical_loss": 3.738825221391676, + "tokens_seen": 779419648 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038577733199598796, + "loss": 2.9809, + "theoretical_loss": 3.738793912191913, + "tokens_seen": 779485184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857673019057172, + "loss": 2.6457, + "theoretical_loss": 3.7387626063613797, + "tokens_seen": 779550720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857572718154463, + "loss": 2.8474, + "theoretical_loss": 3.7387313038994305, + "tokens_seen": 779616256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038574724172517555, + "loss": 2.6877, + "theoretical_loss": 3.7387000048054198, + "tokens_seen": 779681792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038573721163490473, + "loss": 2.9636, + "theoretical_loss": 3.738668709078702, + "tokens_seen": 779747328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857271815446339, + "loss": 2.7967, + "theoretical_loss": 3.7386374167186327, + "tokens_seen": 779812864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 438429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.841843843460083, + "objective/train/theoretical_loss": 3.7386061277245655, + "objective/train/tokens_used": 800338400, + "theoretical_loss": 3.7386061277245655, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857171514543631, + "loss": 3.0358, + "theoretical_loss": 3.7386061277245655, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038570712136409233, + "loss": 3.0418, + "theoretical_loss": 3.7385748420958564, + "tokens_seen": 779943936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038569709127382146, + "loss": 2.6862, + "theoretical_loss": 3.7385435598318604, + "tokens_seen": 780009472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856870611835507, + "loss": 2.7656, + "theoretical_loss": 3.738512280931933, + "tokens_seen": 780075008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856770310932798, + "loss": 2.7398, + "theoretical_loss": 3.73848100539543, + "tokens_seen": 780140544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038566700100300906, + "loss": 2.5016, + "theoretical_loss": 3.7384497332217075, + "tokens_seen": 780206080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038565697091273824, + "loss": 2.6233, + "theoretical_loss": 3.7384184644101204, + "tokens_seen": 780271616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856469408224674, + "loss": 2.7823, + "theoretical_loss": 3.7383871989600257, + "tokens_seen": 780337152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856369107321966, + "loss": 2.7999, + "theoretical_loss": 3.7383559368707804, + "tokens_seen": 780402688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856268806419258, + "loss": 2.6636, + "theoretical_loss": 3.7383246781417396, + "tokens_seen": 780468224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038561685055165496, + "loss": 2.7589, + "theoretical_loss": 3.738293422772261, + "tokens_seen": 780533760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856068204613842, + "loss": 2.6067, + "theoretical_loss": 3.738262170761701, + "tokens_seen": 780599296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855967903711133, + "loss": 3.0154, + "theoretical_loss": 3.738230922109417, + "tokens_seen": 780664832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038558676028084256, + "loss": 2.6455, + "theoretical_loss": 3.7381996768147663, + "tokens_seen": 780730368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855767301905717, + "loss": 2.9596, + "theoretical_loss": 3.738168434877106, + "tokens_seen": 780795904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855667001003009, + "loss": 2.868, + "theoretical_loss": 3.738137196295794, + "tokens_seen": 780861440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855566700100301, + "loss": 2.8184, + "theoretical_loss": 3.738105961070188, + "tokens_seen": 780926976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855466399197593, + "loss": 2.8756, + "theoretical_loss": 3.7380747291996457, + "tokens_seen": 780992512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038553660982948846, + "loss": 2.7564, + "theoretical_loss": 3.738043500683526, + "tokens_seen": 781058048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855265797392177, + "loss": 2.8796, + "theoretical_loss": 3.7380122755211858, + "tokens_seen": 781123584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855165496489468, + "loss": 2.8602, + "theoretical_loss": 3.7379810537119846, + "tokens_seen": 781189120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038550651955867606, + "loss": 2.7505, + "theoretical_loss": 3.7379498352552814, + "tokens_seen": 781254656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854964894684052, + "loss": 2.3087, + "theoretical_loss": 3.7379186201504346, + "tokens_seen": 781320192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854864593781344, + "loss": 2.7144, + "theoretical_loss": 3.737887408396803, + "tokens_seen": 781385728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854764292878636, + "loss": 2.4728, + "theoretical_loss": 3.737856199993746, + "tokens_seen": 781451264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 439045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9254188537597656, + "objective/train/theoretical_loss": 3.737824994940622, + "objective/train/tokens_used": 801976800, + "theoretical_loss": 3.737824994940622, + "tokens_seen": 781516800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854663991975928, + "loss": 2.8778, + "theoretical_loss": 3.737824994940622, + "tokens_seen": 781516800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038545636910732197, + "loss": 2.9068, + "theoretical_loss": 3.7377937932367926, + "tokens_seen": 781582336 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038544633901705115, + "loss": 2.5584, + "theoretical_loss": 3.737762594881616, + "tokens_seen": 781647872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038543630892678033, + "loss": 2.6853, + "theoretical_loss": 3.7377313998744524, + "tokens_seen": 781713408 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038542627883650957, + "loss": 2.6201, + "theoretical_loss": 3.7377002082146618, + "tokens_seen": 781778944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854162487462387, + "loss": 2.8869, + "theoretical_loss": 3.737669019901605, + "tokens_seen": 781844480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038540621865596793, + "loss": 2.7393, + "theoretical_loss": 3.7376378349346417, + "tokens_seen": 781910016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038539618856569705, + "loss": 2.868, + "theoretical_loss": 3.737606653313133, + "tokens_seen": 781975552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853861584754263, + "loss": 2.9731, + "theoretical_loss": 3.737575475036439, + "tokens_seen": 782041088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038537612838515547, + "loss": 2.7581, + "theoretical_loss": 3.737544300103922, + "tokens_seen": 782106624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038536609829488465, + "loss": 2.7906, + "theoretical_loss": 3.7375131285149417, + "tokens_seen": 782172160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038535606820461383, + "loss": 2.7958, + "theoretical_loss": 3.73748196026886, + "tokens_seen": 782237696 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038534603811434307, + "loss": 2.7489, + "theoretical_loss": 3.7374507953650387, + "tokens_seen": 782303232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853360080240722, + "loss": 2.8457, + "theoretical_loss": 3.7374196338028387, + "tokens_seen": 782368768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038532597793380143, + "loss": 2.6944, + "theoretical_loss": 3.7373884755816222, + "tokens_seen": 782434304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038531594784353056, + "loss": 2.95, + "theoretical_loss": 3.7373573207007516, + "tokens_seen": 782499840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853059177532598, + "loss": 2.8403, + "theoretical_loss": 3.737326169159589, + "tokens_seen": 782565376 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385295887662989, + "loss": 2.7555, + "theoretical_loss": 3.7372950209574958, + "tokens_seen": 782630912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038528585757271816, + "loss": 2.7885, + "theoretical_loss": 3.7372638760938353, + "tokens_seen": 782696448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038527582748244734, + "loss": 2.7648, + "theoretical_loss": 3.7372327345679706, + "tokens_seen": 782761984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852657973921765, + "loss": 3.0226, + "theoretical_loss": 3.737201596379263, + "tokens_seen": 782827520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852557673019057, + "loss": 2.7361, + "theoretical_loss": 3.7371704615270778, + "tokens_seen": 782893056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038524573721163493, + "loss": 2.8869, + "theoretical_loss": 3.7371393300107765, + "tokens_seen": 782958592 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038523570712136406, + "loss": 2.7412, + "theoretical_loss": 3.7371082018297233, + "tokens_seen": 783024128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852256770310933, + "loss": 2.7908, + "theoretical_loss": 3.7370770769832813, + "tokens_seen": 783089664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 440183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6950714588165283, + "objective/train/theoretical_loss": 3.7370459554708146, + "objective/train/tokens_used": 803615200, + "theoretical_loss": 3.7370459554708146, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852156469408225, + "loss": 2.6978, + "theoretical_loss": 3.7370459554708146, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038520561685055166, + "loss": 2.9029, + "theoretical_loss": 3.737014837291687, + "tokens_seen": 783220736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038519558676028084, + "loss": 2.8232, + "theoretical_loss": 3.736983722445263, + "tokens_seen": 783286272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038518555667001, + "loss": 2.7811, + "theoretical_loss": 3.7369526109309064, + "tokens_seen": 783351808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851755265797392, + "loss": 2.783, + "theoretical_loss": 3.7369215027479816, + "tokens_seen": 783417344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038516549648946844, + "loss": 2.7497, + "theoretical_loss": 3.736890397895853, + "tokens_seen": 783482880 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038515546639919756, + "loss": 2.7636, + "theoretical_loss": 3.7368592963738863, + "tokens_seen": 783548416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851454363089268, + "loss": 2.6221, + "theoretical_loss": 3.736828198181446, + "tokens_seen": 783613952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851354062186559, + "loss": 3.0129, + "theoretical_loss": 3.736797103317897, + "tokens_seen": 783679488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038512537612838516, + "loss": 2.7719, + "theoretical_loss": 3.7367660117826054, + "tokens_seen": 783745024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851153460381144, + "loss": 2.8759, + "theoretical_loss": 3.736734923574935, + "tokens_seen": 783810560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851053159478435, + "loss": 2.8192, + "theoretical_loss": 3.736703838694254, + "tokens_seen": 783876096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038509528585757276, + "loss": 2.7093, + "theoretical_loss": 3.7366727571399254, + "tokens_seen": 783941632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850852557673019, + "loss": 2.8386, + "theoretical_loss": 3.736641678911318, + "tokens_seen": 784007168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850752256770311, + "loss": 2.8519, + "theoretical_loss": 3.7366106040077955, + "tokens_seen": 784072704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850651955867603, + "loss": 2.9498, + "theoretical_loss": 3.736579532428726, + "tokens_seen": 784138240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850551654964895, + "loss": 2.9198, + "theoretical_loss": 3.7365484641734756, + "tokens_seen": 784203776 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038504513540621866, + "loss": 2.7531, + "theoretical_loss": 3.7365173992414107, + "tokens_seen": 784269312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850351053159479, + "loss": 2.6193, + "theoretical_loss": 3.7364863376318986, + "tokens_seen": 784334848 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385025075225677, + "loss": 3.0243, + "theoretical_loss": 3.736455279344306, + "tokens_seen": 784400384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038501504513540626, + "loss": 2.92, + "theoretical_loss": 3.736424224378, + "tokens_seen": 784465920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850050150451354, + "loss": 2.9541, + "theoretical_loss": 3.736393172732349, + "tokens_seen": 784531456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849949849548646, + "loss": 2.8379, + "theoretical_loss": 3.7363621244067193, + "tokens_seen": 784596992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849849548645938, + "loss": 2.7373, + "theoretical_loss": 3.736331079400479, + "tokens_seen": 784662528 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384974924774323, + "loss": 2.803, + "theoretical_loss": 3.7363000377129962, + "tokens_seen": 784728064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 440843, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.821906566619873, + "objective/train/theoretical_loss": 3.7362689993436398, + "objective/train/tokens_used": 805253600, + "theoretical_loss": 3.7362689993436398, + "tokens_seen": 784793600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038496489468405217, + "loss": 2.7818, + "theoretical_loss": 3.7362689993436398, + "tokens_seen": 784793600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038495486459378135, + "loss": 2.7273, + "theoretical_loss": 3.7362379642917767, + "tokens_seen": 784859136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038494483450351053, + "loss": 2.6269, + "theoretical_loss": 3.736206932556776, + "tokens_seen": 784924672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038493480441323977, + "loss": 2.8092, + "theoretical_loss": 3.7361759041380065, + "tokens_seen": 784990208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849247743229689, + "loss": 2.8882, + "theoretical_loss": 3.736144879034837, + "tokens_seen": 785055744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038491474423269813, + "loss": 2.5399, + "theoretical_loss": 3.736113857246636, + "tokens_seen": 785121280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038490471414242725, + "loss": 2.6503, + "theoretical_loss": 3.7360828387727727, + "tokens_seen": 785186816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848946840521565, + "loss": 2.7638, + "theoretical_loss": 3.736051823612616, + "tokens_seen": 785252352 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038488465396188567, + "loss": 2.6412, + "theoretical_loss": 3.736020811765537, + "tokens_seen": 785317888 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038487462387161485, + "loss": 2.8907, + "theoretical_loss": 3.735989803230904, + "tokens_seen": 785383424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038486459378134403, + "loss": 2.7288, + "theoretical_loss": 3.7359587980080873, + "tokens_seen": 785448960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038485456369107327, + "loss": 2.9444, + "theoretical_loss": 3.7359277960964565, + "tokens_seen": 785514496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848445336008024, + "loss": 2.9635, + "theoretical_loss": 3.735896797495382, + "tokens_seen": 785580032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038483450351053163, + "loss": 2.8273, + "theoretical_loss": 3.735865802204234, + "tokens_seen": 785645568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038482447342026076, + "loss": 2.6847, + "theoretical_loss": 3.735834810222384, + "tokens_seen": 785711104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038481444332999, + "loss": 2.6525, + "theoretical_loss": 3.735803821549201, + "tokens_seen": 785776640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848044132397192, + "loss": 2.6116, + "theoretical_loss": 3.735772836184057, + "tokens_seen": 785842176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038479438314944836, + "loss": 2.8336, + "theoretical_loss": 3.7357418541263234, + "tokens_seen": 785907712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038478435305917754, + "loss": 2.9349, + "theoretical_loss": 3.7357108753753705, + "tokens_seen": 785973248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847743229689067, + "loss": 2.7884, + "theoretical_loss": 3.7356798999305694, + "tokens_seen": 786038784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847642928786359, + "loss": 2.5847, + "theoretical_loss": 3.7356489277912925, + "tokens_seen": 786104320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038475426278836513, + "loss": 2.9119, + "theoretical_loss": 3.7356179589569116, + "tokens_seen": 786169856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038474423269809426, + "loss": 2.5322, + "theoretical_loss": 3.735586993426798, + "tokens_seen": 786235392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847342026078235, + "loss": 2.5874, + "theoretical_loss": 3.735556031200324, + "tokens_seen": 786300928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847241725175527, + "loss": 2.9919, + "theoretical_loss": 3.7355250722768614, + "tokens_seen": 786366464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 442027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.940617561340332, + "objective/train/theoretical_loss": 3.735494116655784, + "objective/train/tokens_used": 806892000, + "theoretical_loss": 3.735494116655784, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038471414242728186, + "loss": 2.9008, + "theoretical_loss": 3.735494116655784, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038470411233701104, + "loss": 2.8244, + "theoretical_loss": 3.735463164336462, + "tokens_seen": 786497536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846940822467402, + "loss": 2.8714, + "theoretical_loss": 3.7354322153182706, + "tokens_seen": 786563072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846840521564694, + "loss": 2.9422, + "theoretical_loss": 3.7354012696005814, + "tokens_seen": 786628608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038467402206619864, + "loss": 2.7046, + "theoretical_loss": 3.735370327182768, + "tokens_seen": 786694144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038466399197592776, + "loss": 2.5513, + "theoretical_loss": 3.7353393880642036, + "tokens_seen": 786759680 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384653961885657, + "loss": 2.9961, + "theoretical_loss": 3.7353084522442614, + "tokens_seen": 786825216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846439317953861, + "loss": 2.7466, + "theoretical_loss": 3.735277519722315, + "tokens_seen": 786890752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038463390170511536, + "loss": 2.7936, + "theoretical_loss": 3.735246590497738, + "tokens_seen": 786956288 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038462387161484454, + "loss": 2.6091, + "theoretical_loss": 3.735215664569905, + "tokens_seen": 787021824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846138415245737, + "loss": 2.5691, + "theoretical_loss": 3.7351847419381894, + "tokens_seen": 787087360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846038114343029, + "loss": 2.775, + "theoretical_loss": 3.7351538226019665, + "tokens_seen": 787152896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845937813440321, + "loss": 2.9211, + "theoretical_loss": 3.7351229065606093, + "tokens_seen": 787218432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038458375125376127, + "loss": 2.8669, + "theoretical_loss": 3.735091993813494, + "tokens_seen": 787283968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845737211634905, + "loss": 2.7544, + "theoretical_loss": 3.7350610843599945, + "tokens_seen": 787349504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038456369107321963, + "loss": 2.7132, + "theoretical_loss": 3.7350301781994855, + "tokens_seen": 787415040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038455366098294886, + "loss": 2.8449, + "theoretical_loss": 3.734999275331343, + "tokens_seen": 787480576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038454363089267805, + "loss": 2.8362, + "theoretical_loss": 3.7349683757549412, + "tokens_seen": 787546112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038453360080240723, + "loss": 2.607, + "theoretical_loss": 3.7349374794696573, + "tokens_seen": 787611648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845235707121364, + "loss": 2.7922, + "theoretical_loss": 3.734906586474865, + "tokens_seen": 787677184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845135406218656, + "loss": 2.7689, + "theoretical_loss": 3.734875696769942, + "tokens_seen": 787742720 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038450351053159477, + "loss": 2.8311, + "theoretical_loss": 3.7348448103542626, + "tokens_seen": 787808256 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384493480441324, + "loss": 2.7133, + "theoretical_loss": 3.734813927227204, + "tokens_seen": 787873792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038448345035105313, + "loss": 2.7541, + "theoretical_loss": 3.734783047388142, + "tokens_seen": 787939328 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038447342026078237, + "loss": 2.7583, + "theoretical_loss": 3.7347521708364537, + "tokens_seen": 788004864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 442587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.145890474319458, + "objective/train/theoretical_loss": 3.7347212975715154, + "objective/train/tokens_used": 808530400, + "theoretical_loss": 3.7347212975715154, + "tokens_seen": 788070400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844633901705115, + "loss": 2.9654, + "theoretical_loss": 3.7347212975715154, + "tokens_seen": 788070400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038445336008024073, + "loss": 2.7115, + "theoretical_loss": 3.734690427592704, + "tokens_seen": 788135936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844433299899699, + "loss": 2.3644, + "theoretical_loss": 3.7346595608993964, + "tokens_seen": 788201472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844332998996991, + "loss": 2.6955, + "theoretical_loss": 3.73462869749097, + "tokens_seen": 788267008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844232698094283, + "loss": 2.8928, + "theoretical_loss": 3.7345978373668025, + "tokens_seen": 788332544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038441323971915745, + "loss": 2.5706, + "theoretical_loss": 3.73456698052627, + "tokens_seen": 788398080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038440320962888664, + "loss": 2.6816, + "theoretical_loss": 3.7345361269687523, + "tokens_seen": 788463616 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038439317953861587, + "loss": 2.6384, + "theoretical_loss": 3.7345052766936253, + "tokens_seen": 788529152 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038438314944834505, + "loss": 2.5129, + "theoretical_loss": 3.7344744297002688, + "tokens_seen": 788594688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038437311935807423, + "loss": 2.6681, + "theoretical_loss": 3.7344435859880596, + "tokens_seen": 788660224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038436308926780347, + "loss": 2.7354, + "theoretical_loss": 3.7344127455563765, + "tokens_seen": 788725760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843530591775326, + "loss": 2.5316, + "theoretical_loss": 3.7343819084045977, + "tokens_seen": 788791296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038434302908726183, + "loss": 2.7936, + "theoretical_loss": 3.734351074532103, + "tokens_seen": 788856832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038433299899699096, + "loss": 2.7816, + "theoretical_loss": 3.7343202439382703, + "tokens_seen": 788922368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843229689067202, + "loss": 2.5337, + "theoretical_loss": 3.734289416622479, + "tokens_seen": 788987904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843129388164494, + "loss": 2.655, + "theoretical_loss": 3.7342585925841085, + "tokens_seen": 789053440 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038430290872617856, + "loss": 2.6479, + "theoretical_loss": 3.734227771822538, + "tokens_seen": 789118976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038429287863590774, + "loss": 2.5965, + "theoretical_loss": 3.7341969543371474, + "tokens_seen": 789184512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842828485456369, + "loss": 2.8551, + "theoretical_loss": 3.7341661401273156, + "tokens_seen": 789250048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842728184553661, + "loss": 2.9743, + "theoretical_loss": 3.734135329192423, + "tokens_seen": 789315584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038426278836509533, + "loss": 2.6725, + "theoretical_loss": 3.73410452153185, + "tokens_seen": 789381120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038425275827482446, + "loss": 2.8066, + "theoretical_loss": 3.734073717144976, + "tokens_seen": 789446656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842427281845537, + "loss": 2.9044, + "theoretical_loss": 3.734042916031182, + "tokens_seen": 789512192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842326980942829, + "loss": 2.7009, + "theoretical_loss": 3.734012118189849, + "tokens_seen": 789577728 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038422266800401206, + "loss": 2.6561, + "theoretical_loss": 3.7339813236203567, + "tokens_seen": 789643264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 443777, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3760197162628174, + "objective/train/theoretical_loss": 3.733950532322087, + "objective/train/tokens_used": 810168800, + "theoretical_loss": 3.733950532322087, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038421263791374124, + "loss": 3.0686, + "theoretical_loss": 3.733950532322087, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842026078234704, + "loss": 2.6925, + "theoretical_loss": 3.73391974429442, + "tokens_seen": 789774336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841925777331996, + "loss": 2.6252, + "theoretical_loss": 3.7338889595367375, + "tokens_seen": 789839872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038418254764292884, + "loss": 2.6106, + "theoretical_loss": 3.733858178048421, + "tokens_seen": 789905408 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038417251755265796, + "loss": 2.6877, + "theoretical_loss": 3.7338273998288516, + "tokens_seen": 789970944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841624874623872, + "loss": 2.6514, + "theoretical_loss": 3.733796624877412, + "tokens_seen": 790036480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841524573721163, + "loss": 2.515, + "theoretical_loss": 3.7337658531934834, + "tokens_seen": 790102016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038414242728184556, + "loss": 2.8697, + "theoretical_loss": 3.733735084776448, + "tokens_seen": 790167552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038413239719157474, + "loss": 2.6622, + "theoretical_loss": 3.733704319625688, + "tokens_seen": 790233088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841223671013039, + "loss": 2.8917, + "theoretical_loss": 3.7336735577405857, + "tokens_seen": 790298624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841123370110331, + "loss": 2.5297, + "theoretical_loss": 3.733642799120524, + "tokens_seen": 790364160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841023069207623, + "loss": 2.7386, + "theoretical_loss": 3.733612043764886, + "tokens_seen": 790429696 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038409227683049147, + "loss": 2.8363, + "theoretical_loss": 3.7335812916730537, + "tokens_seen": 790495232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840822467402207, + "loss": 2.917, + "theoretical_loss": 3.733550542844411, + "tokens_seen": 790560768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038407221664994983, + "loss": 2.7862, + "theoretical_loss": 3.7335197972783405, + "tokens_seen": 790626304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038406218655967907, + "loss": 2.6633, + "theoretical_loss": 3.7334890549742257, + "tokens_seen": 790691840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038405215646940825, + "loss": 2.7952, + "theoretical_loss": 3.7334583159314514, + "tokens_seen": 790757376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038404212637913743, + "loss": 2.7748, + "theoretical_loss": 3.7334275801494, + "tokens_seen": 790822912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840320962888666, + "loss": 2.6345, + "theoretical_loss": 3.733396847627456, + "tokens_seen": 790888448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840220661985958, + "loss": 2.6611, + "theoretical_loss": 3.7333661183650033, + "tokens_seen": 790953984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038401203610832497, + "loss": 2.7081, + "theoretical_loss": 3.7333353923614263, + "tokens_seen": 791019520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840020060180542, + "loss": 2.5762, + "theoretical_loss": 3.7333046696161087, + "tokens_seen": 791085056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038399197592778333, + "loss": 2.9757, + "theoretical_loss": 3.733273950128437, + "tokens_seen": 791150592 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038398194583751257, + "loss": 2.7254, + "theoretical_loss": 3.733243233897794, + "tokens_seen": 791216128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839719157472417, + "loss": 2.7849, + "theoretical_loss": 3.7332125209235656, + "tokens_seen": 791281664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 444379, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.697193145751953, + "objective/train/theoretical_loss": 3.7331818112051365, + "objective/train/tokens_used": 811807200, + "theoretical_loss": 3.7331818112051365, + "tokens_seen": 791347200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038396188565697093, + "loss": 2.9443, + "theoretical_loss": 3.7331818112051365, + "tokens_seen": 791347200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839518555667001, + "loss": 2.7015, + "theoretical_loss": 3.7331511047418924, + "tokens_seen": 791412736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839418254764293, + "loss": 2.8082, + "theoretical_loss": 3.7331204015332187, + "tokens_seen": 791478272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839317953861585, + "loss": 2.6834, + "theoretical_loss": 3.7330897015785003, + "tokens_seen": 791543808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038392176529588765, + "loss": 2.7401, + "theoretical_loss": 3.733059004877124, + "tokens_seen": 791609344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038391173520561684, + "loss": 2.4514, + "theoretical_loss": 3.733028311428475, + "tokens_seen": 791674880 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038390170511534607, + "loss": 2.6914, + "theoretical_loss": 3.73299762123194, + "tokens_seen": 791740416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838916750250752, + "loss": 2.843, + "theoretical_loss": 3.732966934286904, + "tokens_seen": 791805952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038388164493480443, + "loss": 2.693, + "theoretical_loss": 3.7329362505927555, + "tokens_seen": 791871488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838716148445336, + "loss": 2.7452, + "theoretical_loss": 3.73290557014888, + "tokens_seen": 791937024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838615847542628, + "loss": 2.4392, + "theoretical_loss": 3.732874892954663, + "tokens_seen": 792002560 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383851554663992, + "loss": 2.6381, + "theoretical_loss": 3.7328442190094937, + "tokens_seen": 792068096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038384152457372116, + "loss": 2.7508, + "theoretical_loss": 3.732813548312758, + "tokens_seen": 792133632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038383149448345034, + "loss": 2.6207, + "theoretical_loss": 3.7327828808638435, + "tokens_seen": 792199168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838214643931796, + "loss": 2.7213, + "theoretical_loss": 3.732752216662137, + "tokens_seen": 792264704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838114343029087, + "loss": 2.6121, + "theoretical_loss": 3.732721555707027, + "tokens_seen": 792330240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038380140421263794, + "loss": 2.937, + "theoretical_loss": 3.7326908979979008, + "tokens_seen": 792395776 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038379137412236706, + "loss": 2.624, + "theoretical_loss": 3.7326602435341467, + "tokens_seen": 792461312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837813440320963, + "loss": 2.928, + "theoretical_loss": 3.732629592315152, + "tokens_seen": 792526848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837713139418255, + "loss": 2.8103, + "theoretical_loss": 3.732598944340306, + "tokens_seen": 792592384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038376128385155466, + "loss": 2.6622, + "theoretical_loss": 3.732568299608996, + "tokens_seen": 792657920 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038375125376128384, + "loss": 2.7235, + "theoretical_loss": 3.732537658120612, + "tokens_seen": 792723456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837412236710131, + "loss": 2.7727, + "theoretical_loss": 3.7325070198745416, + "tokens_seen": 792788992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837311935807422, + "loss": 2.8658, + "theoretical_loss": 3.732476384870174, + "tokens_seen": 792854528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038372116349047144, + "loss": 2.6207, + "theoretical_loss": 3.732445753106899, + "tokens_seen": 792920064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 445600, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.453542947769165, + "objective/train/theoretical_loss": 3.7324151245841044, + "objective/train/tokens_used": 813445600, + "theoretical_loss": 3.7324151245841044, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038371113340020057, + "loss": 2.8558, + "theoretical_loss": 3.7324151245841044, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837011033099298, + "loss": 2.7573, + "theoretical_loss": 3.7323844993011814, + "tokens_seen": 793051136 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383691073219659, + "loss": 2.7434, + "theoretical_loss": 3.732353877257518, + "tokens_seen": 793116672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038368104312938816, + "loss": 2.7893, + "theoretical_loss": 3.732323258452505, + "tokens_seen": 793182208 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038367101303911735, + "loss": 2.8657, + "theoretical_loss": 3.7322926428855316, + "tokens_seen": 793247744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836609829488465, + "loss": 2.8876, + "theoretical_loss": 3.7322620305559893, + "tokens_seen": 793313280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836509528585757, + "loss": 2.7263, + "theoretical_loss": 3.732231421463266, + "tokens_seen": 793378816 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038364092276830494, + "loss": 2.7745, + "theoretical_loss": 3.7322008156067543, + "tokens_seen": 793444352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836308926780341, + "loss": 2.8643, + "theoretical_loss": 3.732170212985844, + "tokens_seen": 793509888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836208625877633, + "loss": 2.6559, + "theoretical_loss": 3.7321396135999256, + "tokens_seen": 793575424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836108324974925, + "loss": 2.6391, + "theoretical_loss": 3.73210901744839, + "tokens_seen": 793640960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038360080240722167, + "loss": 2.76, + "theoretical_loss": 3.732078424530629, + "tokens_seen": 793706496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835907723169509, + "loss": 2.8954, + "theoretical_loss": 3.732047834846033, + "tokens_seen": 793772032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038358074222668003, + "loss": 2.6983, + "theoretical_loss": 3.732017248393994, + "tokens_seen": 793837568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038357071213640927, + "loss": 2.628, + "theoretical_loss": 3.7319866651739035, + "tokens_seen": 793903104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038356068204613845, + "loss": 2.7902, + "theoretical_loss": 3.7319560851851525, + "tokens_seen": 793968640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038355065195586763, + "loss": 2.6634, + "theoretical_loss": 3.7319255084271337, + "tokens_seen": 794034176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835406218655968, + "loss": 2.8909, + "theoretical_loss": 3.731894934899239, + "tokens_seen": 794099712 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383530591775326, + "loss": 2.7643, + "theoretical_loss": 3.7318643646008605, + "tokens_seen": 794165248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038352056168505517, + "loss": 2.7051, + "theoretical_loss": 3.731833797531391, + "tokens_seen": 794230784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835105315947844, + "loss": 2.8871, + "theoretical_loss": 3.731803233690222, + "tokens_seen": 794296320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038350050150451353, + "loss": 2.9249, + "theoretical_loss": 3.7317726730767475, + "tokens_seen": 794361856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038349047141424277, + "loss": 2.7298, + "theoretical_loss": 3.73174211569036, + "tokens_seen": 794427392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834804413239719, + "loss": 2.8595, + "theoretical_loss": 3.731711561530453, + "tokens_seen": 794492928 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038347041123370113, + "loss": 2.9234, + "theoretical_loss": 3.731681010596418, + "tokens_seen": 794558464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 446700, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2344651222229004, + "objective/train/theoretical_loss": 3.73165046288765, + "objective/train/tokens_used": 815084000, + "theoretical_loss": 3.73165046288765, + "tokens_seen": 794624000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834603811434303, + "loss": 2.5192, + "theoretical_loss": 3.73165046288765, + "tokens_seen": 794624000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834503510531595, + "loss": 2.7426, + "theoretical_loss": 3.731619918403542, + "tokens_seen": 794689536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834403209628887, + "loss": 2.6839, + "theoretical_loss": 3.731589377143488, + "tokens_seen": 794755072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038343029087261786, + "loss": 2.638, + "theoretical_loss": 3.7315588391068815, + "tokens_seen": 794820608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038342026078234704, + "loss": 2.7918, + "theoretical_loss": 3.731528304293117, + "tokens_seen": 794886144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038341023069207627, + "loss": 2.7242, + "theoretical_loss": 3.731497772701588, + "tokens_seen": 794951680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834002006018054, + "loss": 2.7139, + "theoretical_loss": 3.7314672443316894, + "tokens_seen": 795017216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038339017051153463, + "loss": 2.8092, + "theoretical_loss": 3.7314367191828155, + "tokens_seen": 795082752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833801404212638, + "loss": 2.3607, + "theoretical_loss": 3.731406197254361, + "tokens_seen": 795148288 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383370110330993, + "loss": 2.7424, + "theoretical_loss": 3.7313756785457217, + "tokens_seen": 795213824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833600802407222, + "loss": 2.669, + "theoretical_loss": 3.7313451630562913, + "tokens_seen": 795279360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038335005015045136, + "loss": 2.8487, + "theoretical_loss": 3.731314650785465, + "tokens_seen": 795344896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038334002006018054, + "loss": 2.7453, + "theoretical_loss": 3.7312841417326394, + "tokens_seen": 795410432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833299899699098, + "loss": 2.7604, + "theoretical_loss": 3.7312536358972084, + "tokens_seen": 795475968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833199598796389, + "loss": 2.5242, + "theoretical_loss": 3.7312231332785695, + "tokens_seen": 795541504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038330992978936814, + "loss": 2.9914, + "theoretical_loss": 3.731192633876117, + "tokens_seen": 795607040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038329989969909726, + "loss": 2.7639, + "theoretical_loss": 3.7311621376892474, + "tokens_seen": 795672576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832898696088265, + "loss": 2.9645, + "theoretical_loss": 3.7311316447173573, + "tokens_seen": 795738112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832798395185557, + "loss": 2.8367, + "theoretical_loss": 3.731101154959842, + "tokens_seen": 795803648 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038326980942828486, + "loss": 2.8634, + "theoretical_loss": 3.7310706684160992, + "tokens_seen": 795869184 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038325977933801404, + "loss": 2.6604, + "theoretical_loss": 3.731040185085525, + "tokens_seen": 795934720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832497492477433, + "loss": 2.6438, + "theoretical_loss": 3.731009704967516, + "tokens_seen": 796000256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832397191574724, + "loss": 2.8332, + "theoretical_loss": 3.7309792280614698, + "tokens_seen": 796065792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038322968906720164, + "loss": 2.6339, + "theoretical_loss": 3.730948754366783, + "tokens_seen": 796131328 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038321965897693077, + "loss": 2.6891, + "theoretical_loss": 3.730918283882853, + "tokens_seen": 796196864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 447326, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5912489891052246, + "objective/train/theoretical_loss": 3.730887816609077, + "objective/train/tokens_used": 816722400, + "theoretical_loss": 3.730887816609077, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038320962888666, + "loss": 2.8906, + "theoretical_loss": 3.730887816609077, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831995987963892, + "loss": 2.6539, + "theoretical_loss": 3.730857352544853, + "tokens_seen": 796327936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038318956870611836, + "loss": 2.6463, + "theoretical_loss": 3.7308268916895795, + "tokens_seen": 796393472 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038317953861584755, + "loss": 3.0121, + "theoretical_loss": 3.730796434042653, + "tokens_seen": 796459008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831695085255767, + "loss": 2.5218, + "theoretical_loss": 3.730765979603473, + "tokens_seen": 796524544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831594784353059, + "loss": 2.6623, + "theoretical_loss": 3.7307355283714365, + "tokens_seen": 796590080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038314944834503514, + "loss": 2.9648, + "theoretical_loss": 3.7307050803459427, + "tokens_seen": 796655616 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038313941825476427, + "loss": 2.9332, + "theoretical_loss": 3.73067463552639, + "tokens_seen": 796721152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831293881644935, + "loss": 2.6942, + "theoretical_loss": 3.7306441939121773, + "tokens_seen": 796786688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038311935807422263, + "loss": 2.7156, + "theoretical_loss": 3.7306137555027035, + "tokens_seen": 796852224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038310932798395187, + "loss": 2.6772, + "theoretical_loss": 3.7305833202973675, + "tokens_seen": 796917760 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038309929789368105, + "loss": 2.8115, + "theoretical_loss": 3.730552888295569, + "tokens_seen": 796983296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038308926780341023, + "loss": 2.8397, + "theoretical_loss": 3.730522459496707, + "tokens_seen": 797048832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830792377131394, + "loss": 2.7367, + "theoretical_loss": 3.7304920339001812, + "tokens_seen": 797114368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038306920762286865, + "loss": 2.7863, + "theoretical_loss": 3.7304616115053917, + "tokens_seen": 797179904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830591775325978, + "loss": 2.8075, + "theoretical_loss": 3.7304311923117375, + "tokens_seen": 797245440 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383049147442327, + "loss": 2.631, + "theoretical_loss": 3.7304007763186195, + "tokens_seen": 797310976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038303911735205614, + "loss": 2.5869, + "theoretical_loss": 3.730370363525438, + "tokens_seen": 797376512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038302908726178537, + "loss": 2.8324, + "theoretical_loss": 3.7303399539315927, + "tokens_seen": 797442048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038301905717151455, + "loss": 2.586, + "theoretical_loss": 3.7303095475364842, + "tokens_seen": 797507584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038300902708124373, + "loss": 2.7626, + "theoretical_loss": 3.730279144339514, + "tokens_seen": 797573120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829989969909729, + "loss": 2.8478, + "theoretical_loss": 3.7302487443400825, + "tokens_seen": 797638656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829889669007021, + "loss": 2.798, + "theoretical_loss": 3.7302183475375905, + "tokens_seen": 797704192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829789368104313, + "loss": 2.6434, + "theoretical_loss": 3.7301879539314395, + "tokens_seen": 797769728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829689067201605, + "loss": 2.6643, + "theoretical_loss": 3.730157563521031, + "tokens_seen": 797835264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 447858, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1472742557525635, + "objective/train/theoretical_loss": 3.730127176305765, + "objective/train/tokens_used": 818360800, + "theoretical_loss": 3.730127176305765, + "tokens_seen": 797900800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038295887662988964, + "loss": 2.845, + "theoretical_loss": 3.730127176305765, + "tokens_seen": 797900800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829488465396189, + "loss": 2.7808, + "theoretical_loss": 3.730096792285046, + "tokens_seen": 797966336 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382938816449348, + "loss": 2.6188, + "theoretical_loss": 3.7300664114582736, + "tokens_seen": 798031872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038292878635907724, + "loss": 2.8079, + "theoretical_loss": 3.7300360338248506, + "tokens_seen": 798097408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829187562688064, + "loss": 2.9146, + "theoretical_loss": 3.7300056593841795, + "tokens_seen": 798162944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829087261785356, + "loss": 2.9053, + "theoretical_loss": 3.7299752881356616, + "tokens_seen": 798228480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828986960882648, + "loss": 2.5042, + "theoretical_loss": 3.7299449200787, + "tokens_seen": 798294016 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382888665997994, + "loss": 2.6706, + "theoretical_loss": 3.729914555212698, + "tokens_seen": 798359552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828786359077232, + "loss": 2.3704, + "theoretical_loss": 3.7298841935370572, + "tokens_seen": 798425088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828686058174524, + "loss": 2.6598, + "theoretical_loss": 3.729853835051181, + "tokens_seen": 798490624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038285857572718156, + "loss": 2.8312, + "theoretical_loss": 3.729823479754473, + "tokens_seen": 798556160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038284854563691074, + "loss": 2.8285, + "theoretical_loss": 3.729793127646336, + "tokens_seen": 798621696 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038283851554664, + "loss": 2.6438, + "theoretical_loss": 3.7297627787261733, + "tokens_seen": 798687232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828284854563691, + "loss": 2.6833, + "theoretical_loss": 3.729732432993389, + "tokens_seen": 798752768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038281845536609834, + "loss": 2.7391, + "theoretical_loss": 3.729702090447387, + "tokens_seen": 798818304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038280842527582746, + "loss": 2.4969, + "theoretical_loss": 3.7296717510875705, + "tokens_seen": 798883840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827983951855567, + "loss": 2.7266, + "theoretical_loss": 3.7296414149133437, + "tokens_seen": 798949376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827883650952859, + "loss": 2.7095, + "theoretical_loss": 3.7296110819241117, + "tokens_seen": 799014912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038277833500501506, + "loss": 2.6529, + "theoretical_loss": 3.729580752119278, + "tokens_seen": 799080448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038276830491474424, + "loss": 2.774, + "theoretical_loss": 3.729550425498248, + "tokens_seen": 799145984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827582748244735, + "loss": 2.671, + "theoretical_loss": 3.729520102060425, + "tokens_seen": 799211520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827482447342026, + "loss": 2.6735, + "theoretical_loss": 3.7294897818052157, + "tokens_seen": 799277056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038273821464393184, + "loss": 2.6403, + "theoretical_loss": 3.729459464732024, + "tokens_seen": 799342592 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038272818455366097, + "loss": 2.7248, + "theoretical_loss": 3.7294291508402555, + "tokens_seen": 799408128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827181544633902, + "loss": 2.765, + "theoretical_loss": 3.729398840129315, + "tokens_seen": 799473664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 448758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5953433513641357, + "objective/train/theoretical_loss": 3.729368532598609, + "objective/train/tokens_used": 819999200, + "theoretical_loss": 3.729368532598609, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827081243731194, + "loss": 2.7885, + "theoretical_loss": 3.729368532598609, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038269809428284856, + "loss": 2.8063, + "theoretical_loss": 3.729338228247543, + "tokens_seen": 799604736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038268806419257775, + "loss": 2.7712, + "theoretical_loss": 3.7293079270755216, + "tokens_seen": 799670272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826780341023069, + "loss": 2.814, + "theoretical_loss": 3.7292776290819525, + "tokens_seen": 799735808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826680040120361, + "loss": 2.8649, + "theoretical_loss": 3.729247334266241, + "tokens_seen": 799801344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038265797392176534, + "loss": 2.8075, + "theoretical_loss": 3.7292170426277935, + "tokens_seen": 799866880 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038264794383149447, + "loss": 2.6499, + "theoretical_loss": 3.7291867541660166, + "tokens_seen": 799932416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826379137412237, + "loss": 2.7997, + "theoretical_loss": 3.729156468880317, + "tokens_seen": 799997952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038262788365095283, + "loss": 2.6938, + "theoretical_loss": 3.729126186770101, + "tokens_seen": 800063488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038261785356068207, + "loss": 2.9072, + "theoretical_loss": 3.7290959078347763, + "tokens_seen": 800129024 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038260782347041125, + "loss": 2.6993, + "theoretical_loss": 3.72906563207375, + "tokens_seen": 800194560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038259779338014043, + "loss": 2.5006, + "theoretical_loss": 3.7290353594864287, + "tokens_seen": 800260096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825877632898696, + "loss": 3.0636, + "theoretical_loss": 3.7290050900722203, + "tokens_seen": 800325632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038257773319959885, + "loss": 2.8181, + "theoretical_loss": 3.7289748238305327, + "tokens_seen": 800391168 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382567703109328, + "loss": 2.6539, + "theoretical_loss": 3.7289445607607727, + "tokens_seen": 800456704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825576730190572, + "loss": 2.9976, + "theoretical_loss": 3.728914300862349, + "tokens_seen": 800522240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038254764292878634, + "loss": 2.7945, + "theoretical_loss": 3.72888404413467, + "tokens_seen": 800587776 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038253761283851557, + "loss": 2.7775, + "theoretical_loss": 3.7288537905771433, + "tokens_seen": 800653312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038252758274824475, + "loss": 2.5832, + "theoretical_loss": 3.728823540189177, + "tokens_seen": 800718848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038251755265797393, + "loss": 2.8424, + "theoretical_loss": 3.728793292970181, + "tokens_seen": 800784384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825075225677031, + "loss": 3.0617, + "theoretical_loss": 3.7287630489195625, + "tokens_seen": 800849920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824974924774323, + "loss": 2.771, + "theoretical_loss": 3.728732808036731, + "tokens_seen": 800915456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824874623871615, + "loss": 2.8826, + "theoretical_loss": 3.7287025703210954, + "tokens_seen": 800980992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824774322968907, + "loss": 2.9558, + "theoretical_loss": 3.7286723357720657, + "tokens_seen": 801046528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038246740220661984, + "loss": 2.7418, + "theoretical_loss": 3.72864210438905, + "tokens_seen": 801112064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 449233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.920238733291626, + "objective/train/theoretical_loss": 3.728611876171459, + "objective/train/tokens_used": 821637600, + "theoretical_loss": 3.728611876171459, + "tokens_seen": 801177600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824573721163491, + "loss": 2.7952, + "theoretical_loss": 3.728611876171459, + "tokens_seen": 801177600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824473420260782, + "loss": 2.6668, + "theoretical_loss": 3.7285816511187013, + "tokens_seen": 801243136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038243731193580744, + "loss": 2.7591, + "theoretical_loss": 3.7285514292301873, + "tokens_seen": 801308672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824272818455366, + "loss": 2.7631, + "theoretical_loss": 3.728521210505327, + "tokens_seen": 801374208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824172517552658, + "loss": 2.7571, + "theoretical_loss": 3.7284909949435305, + "tokens_seen": 801439744 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382407221664995, + "loss": 2.655, + "theoretical_loss": 3.728460782544208, + "tokens_seen": 801505280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823971915747242, + "loss": 2.884, + "theoretical_loss": 3.7284305733067704, + "tokens_seen": 801570816 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038238716148445334, + "loss": 2.7354, + "theoretical_loss": 3.728400367230627, + "tokens_seen": 801636352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823771313941826, + "loss": 2.7962, + "theoretical_loss": 3.7283701643151903, + "tokens_seen": 801701888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823671013039117, + "loss": 2.7577, + "theoretical_loss": 3.7283399645598703, + "tokens_seen": 801767424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038235707121364094, + "loss": 2.929, + "theoretical_loss": 3.728309767964078, + "tokens_seen": 801832960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823470411233701, + "loss": 2.6252, + "theoretical_loss": 3.728279574527225, + "tokens_seen": 801898496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823370110330993, + "loss": 2.6991, + "theoretical_loss": 3.7282493842487225, + "tokens_seen": 801964032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823269809428285, + "loss": 2.6825, + "theoretical_loss": 3.7282191971279826, + "tokens_seen": 802029568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038231695085255766, + "loss": 2.6514, + "theoretical_loss": 3.7281890131644158, + "tokens_seen": 802095104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038230692076228685, + "loss": 2.7606, + "theoretical_loss": 3.728158832357435, + "tokens_seen": 802160640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822968906720161, + "loss": 2.7181, + "theoretical_loss": 3.7281286547064525, + "tokens_seen": 802226176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822868605817452, + "loss": 2.7173, + "theoretical_loss": 3.7280984802108796, + "tokens_seen": 802291712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038227683049147444, + "loss": 2.7012, + "theoretical_loss": 3.7280683088701294, + "tokens_seen": 802357248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038226680040120357, + "loss": 2.6665, + "theoretical_loss": 3.728038140683614, + "tokens_seen": 802422784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822567703109328, + "loss": 2.7573, + "theoretical_loss": 3.728007975650746, + "tokens_seen": 802488320 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382246740220662, + "loss": 2.8618, + "theoretical_loss": 3.727977813770938, + "tokens_seen": 802553856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038223671013039117, + "loss": 2.6473, + "theoretical_loss": 3.7279476550436037, + "tokens_seen": 802619392 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038222668004012035, + "loss": 2.6769, + "theoretical_loss": 3.727917499468156, + "tokens_seen": 802684928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822166499498496, + "loss": 2.8372, + "theoretical_loss": 3.7278873470440077, + "tokens_seen": 802750464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 450645, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0350584983825684, + "objective/train/theoretical_loss": 3.7278571977705734, + "objective/train/tokens_used": 823276000, + "theoretical_loss": 3.7278571977705734, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822066198595787, + "loss": 2.6396, + "theoretical_loss": 3.7278571977705734, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038219658976930795, + "loss": 2.7545, + "theoretical_loss": 3.727827051647265, + "tokens_seen": 802881536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821865596790371, + "loss": 2.7913, + "theoretical_loss": 3.727796908673498, + "tokens_seen": 802947072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821765295887663, + "loss": 2.7719, + "theoretical_loss": 3.727766768848685, + "tokens_seen": 803012608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821664994984955, + "loss": 2.6693, + "theoretical_loss": 3.727736632172241, + "tokens_seen": 803078144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038215646940822467, + "loss": 2.6396, + "theoretical_loss": 3.7277064986435793, + "tokens_seen": 803143680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038214643931795385, + "loss": 2.7354, + "theoretical_loss": 3.7276763682621157, + "tokens_seen": 803209216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038213640922768303, + "loss": 2.6416, + "theoretical_loss": 3.7276462410272635, + "tokens_seen": 803274752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038212637913741227, + "loss": 2.9243, + "theoretical_loss": 3.727616116938438, + "tokens_seen": 803340288 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038211634904714145, + "loss": 2.6483, + "theoretical_loss": 3.7275859959950535, + "tokens_seen": 803405824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038210631895687063, + "loss": 2.7773, + "theoretical_loss": 3.727555878196526, + "tokens_seen": 803471360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820962888665998, + "loss": 2.7851, + "theoretical_loss": 3.7275257635422694, + "tokens_seen": 803536896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038208625877632905, + "loss": 2.5762, + "theoretical_loss": 3.7274956520317, + "tokens_seen": 803602432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820762286860582, + "loss": 2.8452, + "theoretical_loss": 3.727465543664233, + "tokens_seen": 803667968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820661985957874, + "loss": 2.7465, + "theoretical_loss": 3.7274354384392847, + "tokens_seen": 803733504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038205616850551654, + "loss": 2.6667, + "theoretical_loss": 3.7274053363562696, + "tokens_seen": 803799040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038204613841524577, + "loss": 2.8064, + "theoretical_loss": 3.7273752374146043, + "tokens_seen": 803864576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038203610832497495, + "loss": 2.7668, + "theoretical_loss": 3.7273451416137053, + "tokens_seen": 803930112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038202607823470413, + "loss": 2.7471, + "theoretical_loss": 3.7273150489529883, + "tokens_seen": 803995648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820160481444333, + "loss": 2.863, + "theoretical_loss": 3.72728495943187, + "tokens_seen": 804061184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820060180541625, + "loss": 2.7216, + "theoretical_loss": 3.727254873049767, + "tokens_seen": 804126720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819959879638917, + "loss": 2.6481, + "theoretical_loss": 3.7272247898060966, + "tokens_seen": 804192256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819859578736209, + "loss": 2.6998, + "theoretical_loss": 3.7271947097002744, + "tokens_seen": 804257792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038197592778335004, + "loss": 2.9825, + "theoretical_loss": 3.727164632731718, + "tokens_seen": 804323328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819658976930793, + "loss": 3.0266, + "theoretical_loss": 3.727134558899845, + "tokens_seen": 804388864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 451555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6607892513275146, + "objective/train/theoretical_loss": 3.7271044882040725, + "objective/train/tokens_used": 824914400, + "theoretical_loss": 3.7271044882040725, + "tokens_seen": 804454400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819558676028084, + "loss": 2.6691, + "theoretical_loss": 3.7271044882040725, + "tokens_seen": 804454400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038194583751253764, + "loss": 2.9954, + "theoretical_loss": 3.7270744206438184, + "tokens_seen": 804519936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819358074222668, + "loss": 3.0286, + "theoretical_loss": 3.7270443562184994, + "tokens_seen": 804585472 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381925777331996, + "loss": 2.8963, + "theoretical_loss": 3.7270142949275344, + "tokens_seen": 804651008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819157472417252, + "loss": 2.7464, + "theoretical_loss": 3.726984236770341, + "tokens_seen": 804716544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819057171514544, + "loss": 2.6961, + "theoretical_loss": 3.7269541817463363, + "tokens_seen": 804782080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038189568706118354, + "loss": 2.8959, + "theoretical_loss": 3.7269241298549405, + "tokens_seen": 804847616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818856569709128, + "loss": 2.8578, + "theoretical_loss": 3.726894081095571, + "tokens_seen": 804913152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818756268806419, + "loss": 2.7054, + "theoretical_loss": 3.7268640354676466, + "tokens_seen": 804978688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038186559679037114, + "loss": 2.7952, + "theoretical_loss": 3.726833992970586, + "tokens_seen": 805044224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818555667001003, + "loss": 2.8643, + "theoretical_loss": 3.726803953603808, + "tokens_seen": 805109760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818455366098295, + "loss": 2.9446, + "theoretical_loss": 3.7267739173667316, + "tokens_seen": 805175296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818355065195587, + "loss": 3.0333, + "theoretical_loss": 3.7267438842587763, + "tokens_seen": 805240832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038182547642928786, + "loss": 2.7721, + "theoretical_loss": 3.726713854279361, + "tokens_seen": 805306368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038181544633901705, + "loss": 2.7606, + "theoretical_loss": 3.7266838274279066, + "tokens_seen": 805371904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818054162487463, + "loss": 2.8632, + "theoretical_loss": 3.7266538037038313, + "tokens_seen": 805437440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817953861584754, + "loss": 2.8549, + "theoretical_loss": 3.7266237831065556, + "tokens_seen": 805502976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038178535606820464, + "loss": 2.9827, + "theoretical_loss": 3.7265937656354993, + "tokens_seen": 805568512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038177532597793377, + "loss": 2.5919, + "theoretical_loss": 3.7265637512900827, + "tokens_seen": 805634048 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381765295887663, + "loss": 2.5312, + "theoretical_loss": 3.7265337400697263, + "tokens_seen": 805699584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817552657973922, + "loss": 2.6245, + "theoretical_loss": 3.7265037319738497, + "tokens_seen": 805765120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038174523570712137, + "loss": 2.6506, + "theoretical_loss": 3.7264737270018746, + "tokens_seen": 805830656 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038173520561685055, + "loss": 2.7696, + "theoretical_loss": 3.7264437251532216, + "tokens_seen": 805896192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817251755265798, + "loss": 2.8109, + "theoretical_loss": 3.7264137264273107, + "tokens_seen": 805961728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817151454363089, + "loss": 2.7733, + "theoretical_loss": 3.726383730823564, + "tokens_seen": 806027264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 452218, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7736008167266846, + "objective/train/theoretical_loss": 3.7263537383414023, + "objective/train/tokens_used": 826552800, + "theoretical_loss": 3.7263537383414023, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038170511534603815, + "loss": 2.6347, + "theoretical_loss": 3.7263537383414023, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816950852557673, + "loss": 2.6228, + "theoretical_loss": 3.7263237489802474, + "tokens_seen": 806158336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816850551654965, + "loss": 2.7033, + "theoretical_loss": 3.72629376273952, + "tokens_seen": 806223872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816750250752257, + "loss": 2.6791, + "theoretical_loss": 3.726263779618643, + "tokens_seen": 806289408 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038166499498495487, + "loss": 2.7004, + "theoretical_loss": 3.726233799617037, + "tokens_seen": 806354944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038165496489468405, + "loss": 2.6775, + "theoretical_loss": 3.7262038227341248, + "tokens_seen": 806420480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038164493480441323, + "loss": 2.7478, + "theoretical_loss": 3.726173848969329, + "tokens_seen": 806486016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816349047141424, + "loss": 2.7054, + "theoretical_loss": 3.72614387832207, + "tokens_seen": 806551552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038162487462387165, + "loss": 2.9783, + "theoretical_loss": 3.7261139107917725, + "tokens_seen": 806617088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816148445336008, + "loss": 2.7914, + "theoretical_loss": 3.726083946377858, + "tokens_seen": 806682624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038160481444333, + "loss": 2.67, + "theoretical_loss": 3.726053985079749, + "tokens_seen": 806748160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038159478435305914, + "loss": 2.6837, + "theoretical_loss": 3.7260240268968694, + "tokens_seen": 806813696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815847542627884, + "loss": 2.6676, + "theoretical_loss": 3.7259940718286417, + "tokens_seen": 806879232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038157472417251756, + "loss": 2.7729, + "theoretical_loss": 3.7259641198744893, + "tokens_seen": 806944768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038156469408224674, + "loss": 2.7676, + "theoretical_loss": 3.7259341710338356, + "tokens_seen": 807010304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815546639919759, + "loss": 2.7686, + "theoretical_loss": 3.7259042253061043, + "tokens_seen": 807075840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038154463390170515, + "loss": 2.8106, + "theoretical_loss": 3.725874282690718, + "tokens_seen": 807141376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815346038114343, + "loss": 2.7067, + "theoretical_loss": 3.7258443431871022, + "tokens_seen": 807206912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815245737211635, + "loss": 2.9003, + "theoretical_loss": 3.7258144067946803, + "tokens_seen": 807272448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038151454363089264, + "loss": 2.8686, + "theoretical_loss": 3.725784473512876, + "tokens_seen": 807337984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815045135406219, + "loss": 2.6977, + "theoretical_loss": 3.7257545433411137, + "tokens_seen": 807403520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038149448345035106, + "loss": 2.5566, + "theoretical_loss": 3.7257246162788187, + "tokens_seen": 807469056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038148445336008024, + "loss": 2.6849, + "theoretical_loss": 3.7256946923254146, + "tokens_seen": 807534592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814744232698094, + "loss": 2.8655, + "theoretical_loss": 3.725664771480327, + "tokens_seen": 807600128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814643931795386, + "loss": 2.8026, + "theoretical_loss": 3.7256348537429798, + "tokens_seen": 807665664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 453465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5026791095733643, + "objective/train/theoretical_loss": 3.7256049391127988, + "objective/train/tokens_used": 828191200, + "theoretical_loss": 3.7256049391127988, + "tokens_seen": 807731200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814543630892678, + "loss": 2.7585, + "theoretical_loss": 3.7256049391127988, + "tokens_seen": 807731200 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381444332998997, + "loss": 2.5317, + "theoretical_loss": 3.7255750275892097, + "tokens_seen": 807796736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038143430290872614, + "loss": 2.5894, + "theoretical_loss": 3.725545119171637, + "tokens_seen": 807862272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814242728184554, + "loss": 2.8322, + "theoretical_loss": 3.7255152138595067, + "tokens_seen": 807927808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038141424272818456, + "loss": 2.89, + "theoretical_loss": 3.7254853116522444, + "tokens_seen": 807993344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038140421263791374, + "loss": 2.7496, + "theoretical_loss": 3.7254554125492754, + "tokens_seen": 808058880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813941825476429, + "loss": 2.8516, + "theoretical_loss": 3.7254255165500267, + "tokens_seen": 808124416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813841524573721, + "loss": 2.6526, + "theoretical_loss": 3.725395623653924, + "tokens_seen": 808189952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038137412236710134, + "loss": 2.6249, + "theoretical_loss": 3.7253657338603934, + "tokens_seen": 808255488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813640922768305, + "loss": 2.8297, + "theoretical_loss": 3.7253358471688616, + "tokens_seen": 808321024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813540621865597, + "loss": 2.6693, + "theoretical_loss": 3.725305963578755, + "tokens_seen": 808386560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813440320962889, + "loss": 2.7322, + "theoretical_loss": 3.725276083089501, + "tokens_seen": 808452096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038133400200601806, + "loss": 2.7031, + "theoretical_loss": 3.725246205700526, + "tokens_seen": 808517632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038132397191574725, + "loss": 2.6764, + "theoretical_loss": 3.7252163314112567, + "tokens_seen": 808583168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813139418254765, + "loss": 2.6776, + "theoretical_loss": 3.7251864602211207, + "tokens_seen": 808648704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813039117352056, + "loss": 2.4349, + "theoretical_loss": 3.7251565921295455, + "tokens_seen": 808714240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038129388164493484, + "loss": 2.6752, + "theoretical_loss": 3.725126727135959, + "tokens_seen": 808779776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038128385155466397, + "loss": 2.7746, + "theoretical_loss": 3.7250968652397876, + "tokens_seen": 808845312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812738214643932, + "loss": 2.8142, + "theoretical_loss": 3.7250670064404603, + "tokens_seen": 808910848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812637913741224, + "loss": 2.9367, + "theoretical_loss": 3.725037150737405, + "tokens_seen": 808976384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038125376128385157, + "loss": 2.7664, + "theoretical_loss": 3.725007298130049, + "tokens_seen": 809041920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038124373119358075, + "loss": 2.6014, + "theoretical_loss": 3.724977448617821, + "tokens_seen": 809107456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038123370110331, + "loss": 2.67, + "theoretical_loss": 3.72494760220015, + "tokens_seen": 809172992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812236710130391, + "loss": 2.9132, + "theoretical_loss": 3.724917758876464, + "tokens_seen": 809238528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038121364092276835, + "loss": 2.8343, + "theoretical_loss": 3.724887918646192, + "tokens_seen": 809304064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 454216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.918653726577759, + "objective/train/theoretical_loss": 3.7248580815087626, + "objective/train/tokens_used": 829829600, + "theoretical_loss": 3.7248580815087626, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812036108324975, + "loss": 2.5661, + "theoretical_loss": 3.7248580815087626, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811935807422267, + "loss": 2.5607, + "theoretical_loss": 3.724828247463605, + "tokens_seen": 809435136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811835506519559, + "loss": 2.6057, + "theoretical_loss": 3.7247984165101484, + "tokens_seen": 809500672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038117352056168507, + "loss": 2.5209, + "theoretical_loss": 3.724768588647822, + "tokens_seen": 809566208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038116349047141425, + "loss": 2.658, + "theoretical_loss": 3.724738763876056, + "tokens_seen": 809631744 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038115346038114343, + "loss": 2.7838, + "theoretical_loss": 3.724708942194279, + "tokens_seen": 809697280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811434302908726, + "loss": 2.648, + "theoretical_loss": 3.7246791236019217, + "tokens_seen": 809762816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038113340020060185, + "loss": 2.7635, + "theoretical_loss": 3.7246493080984133, + "tokens_seen": 809828352 + }, + { + "epoch": 0.25, + "learning_rate": 0.000381123370110331, + "loss": 2.8069, + "theoretical_loss": 3.724619495683185, + "tokens_seen": 809893888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811133400200602, + "loss": 2.812, + "theoretical_loss": 3.724589686355665, + "tokens_seen": 809959424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038110330992978934, + "loss": 2.706, + "theoretical_loss": 3.7245598801152857, + "tokens_seen": 810024960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810932798395186, + "loss": 2.8072, + "theoretical_loss": 3.7245300769614773, + "tokens_seen": 810090496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038108324974924776, + "loss": 2.7348, + "theoretical_loss": 3.7245002768936697, + "tokens_seen": 810156032 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038107321965897694, + "loss": 2.8901, + "theoretical_loss": 3.724470479911295, + "tokens_seen": 810221568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810631895687061, + "loss": 3.0223, + "theoretical_loss": 3.724440686013782, + "tokens_seen": 810287104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038105315947843535, + "loss": 2.6307, + "theoretical_loss": 3.724410895200565, + "tokens_seen": 810352640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810431293881645, + "loss": 2.8834, + "theoretical_loss": 3.724381107471072, + "tokens_seen": 810418176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810330992978937, + "loss": 2.4979, + "theoretical_loss": 3.724351322824737, + "tokens_seen": 810483712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038102306920762284, + "loss": 2.7276, + "theoretical_loss": 3.724321541260991, + "tokens_seen": 810549248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810130391173521, + "loss": 2.6296, + "theoretical_loss": 3.7242917627792647, + "tokens_seen": 810614784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038100300902708126, + "loss": 2.4256, + "theoretical_loss": 3.7242619873789913, + "tokens_seen": 810680320 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038099297893681044, + "loss": 2.7176, + "theoretical_loss": 3.724232215059603, + "tokens_seen": 810745856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809829488465396, + "loss": 2.7171, + "theoretical_loss": 3.72420244582053, + "tokens_seen": 810811392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809729187562688, + "loss": 2.6742, + "theoretical_loss": 3.7241726796612067, + "tokens_seen": 810876928 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380962888665998, + "loss": 2.5437, + "theoretical_loss": 3.724142916581065, + "tokens_seen": 810942464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 455558, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9195406436920166, + "objective/train/theoretical_loss": 3.724113156579537, + "objective/train/tokens_used": 831468000, + "theoretical_loss": 3.724113156579537, + "tokens_seen": 811008000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809528585757272, + "loss": 2.731, + "theoretical_loss": 3.724113156579537, + "tokens_seen": 811008000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038094282848545635, + "loss": 2.6566, + "theoretical_loss": 3.7240833996560565, + "tokens_seen": 811073536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809327983951856, + "loss": 2.8637, + "theoretical_loss": 3.7240536458100557, + "tokens_seen": 811139072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038092276830491476, + "loss": 2.8568, + "theoretical_loss": 3.7240238950409683, + "tokens_seen": 811204608 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038091273821464394, + "loss": 2.5492, + "theoretical_loss": 3.723994147348227, + "tokens_seen": 811270144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809027081243731, + "loss": 2.6746, + "theoretical_loss": 3.723964402731266, + "tokens_seen": 811335680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808926780341023, + "loss": 2.7153, + "theoretical_loss": 3.723934661189518, + "tokens_seen": 811401216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808826479438315, + "loss": 2.8033, + "theoretical_loss": 3.7239049227224172, + "tokens_seen": 811466752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808726178535607, + "loss": 2.917, + "theoretical_loss": 3.723875187329398, + "tokens_seen": 811532288 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038086258776328985, + "loss": 2.8844, + "theoretical_loss": 3.7238454550098927, + "tokens_seen": 811597824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808525576730191, + "loss": 2.4595, + "theoretical_loss": 3.7238157257633366, + "tokens_seen": 811663360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808425275827482, + "loss": 2.9098, + "theoretical_loss": 3.7237859995891647, + "tokens_seen": 811728896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038083249749247745, + "loss": 2.7129, + "theoretical_loss": 3.72375627648681, + "tokens_seen": 811794432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808224674022066, + "loss": 2.668, + "theoretical_loss": 3.7237265564557083, + "tokens_seen": 811859968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808124373119358, + "loss": 2.6803, + "theoretical_loss": 3.723696839495294, + "tokens_seen": 811925504 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380802407221665, + "loss": 2.7145, + "theoretical_loss": 3.723667125605002, + "tokens_seen": 811991040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038079237713139417, + "loss": 2.7764, + "theoretical_loss": 3.723637414784267, + "tokens_seen": 812056576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038078234704112335, + "loss": 2.6353, + "theoretical_loss": 3.723607707032525, + "tokens_seen": 812122112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807723169508526, + "loss": 2.8169, + "theoretical_loss": 3.7235780023492104, + "tokens_seen": 812187648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807622868605817, + "loss": 2.9368, + "theoretical_loss": 3.7235483007337593, + "tokens_seen": 812253184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038075225677031095, + "loss": 2.8544, + "theoretical_loss": 3.7235186021856075, + "tokens_seen": 812318720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038074222668004013, + "loss": 2.5776, + "theoretical_loss": 3.7234889067041905, + "tokens_seen": 812384256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807321965897693, + "loss": 2.8329, + "theoretical_loss": 3.723459214288944, + "tokens_seen": 812449792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807221664994985, + "loss": 2.7954, + "theoretical_loss": 3.723429524939305, + "tokens_seen": 812515328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807121364092277, + "loss": 2.5979, + "theoretical_loss": 3.723399838654709, + "tokens_seen": 812580864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 456229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.552311897277832, + "objective/train/theoretical_loss": 3.7233701554345924, + "objective/train/tokens_used": 833106400, + "theoretical_loss": 3.7233701554345924, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038070210631895685, + "loss": 2.6729, + "theoretical_loss": 3.7233701554345924, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806920762286861, + "loss": 2.6202, + "theoretical_loss": 3.723340475278392, + "tokens_seen": 812711936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806820461384152, + "loss": 2.6352, + "theoretical_loss": 3.7233107981855453, + "tokens_seen": 812777472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038067201604814445, + "loss": 2.9071, + "theoretical_loss": 3.7232811241554877, + "tokens_seen": 812843008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806619859578736, + "loss": 2.8939, + "theoretical_loss": 3.7232514531876566, + "tokens_seen": 812908544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806519558676028, + "loss": 2.5899, + "theoretical_loss": 3.72322178528149, + "tokens_seen": 812974080 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380641925777332, + "loss": 2.7315, + "theoretical_loss": 3.7231921204364244, + "tokens_seen": 813039616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806318956870612, + "loss": 2.8286, + "theoretical_loss": 3.7231624586518977, + "tokens_seen": 813105152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806218655967904, + "loss": 2.5656, + "theoretical_loss": 3.723132799927347, + "tokens_seen": 813170688 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038061183550651954, + "loss": 2.7309, + "theoretical_loss": 3.7231031442622107, + "tokens_seen": 813236224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806018054162488, + "loss": 2.7168, + "theoretical_loss": 3.723073491655926, + "tokens_seen": 813301760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038059177532597796, + "loss": 2.8353, + "theoretical_loss": 3.7230438421079315, + "tokens_seen": 813367296 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038058174523570714, + "loss": 2.542, + "theoretical_loss": 3.7230141956176652, + "tokens_seen": 813432832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805717151454363, + "loss": 2.7511, + "theoretical_loss": 3.722984552184565, + "tokens_seen": 813498368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038056168505516555, + "loss": 2.5704, + "theoretical_loss": 3.7229549118080705, + "tokens_seen": 813563904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805516549648947, + "loss": 2.5106, + "theoretical_loss": 3.7229252744876193, + "tokens_seen": 813629440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805416248746239, + "loss": 2.8, + "theoretical_loss": 3.7228956402226503, + "tokens_seen": 813694976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038053159478435304, + "loss": 2.5614, + "theoretical_loss": 3.722866009012603, + "tokens_seen": 813760512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805215646940823, + "loss": 2.7477, + "theoretical_loss": 3.7228363808569163, + "tokens_seen": 813826048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038051153460381146, + "loss": 2.7359, + "theoretical_loss": 3.722806755755029, + "tokens_seen": 813891584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038050150451354064, + "loss": 2.8882, + "theoretical_loss": 3.722777133706381, + "tokens_seen": 813957120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804914744232698, + "loss": 2.7194, + "theoretical_loss": 3.7227475147104117, + "tokens_seen": 814022656 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380481444332999, + "loss": 2.6075, + "theoretical_loss": 3.7227178987665606, + "tokens_seen": 814088192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804714142427282, + "loss": 2.7222, + "theoretical_loss": 3.722688285874267, + "tokens_seen": 814153728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804613841524574, + "loss": 2.6867, + "theoretical_loss": 3.7226586760329727, + "tokens_seen": 814219264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 457261, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5911662578582764, + "objective/train/theoretical_loss": 3.7226290692421165, + "objective/train/tokens_used": 834744800, + "theoretical_loss": 3.7226290692421165, + "tokens_seen": 814284800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038045135406218655, + "loss": 2.6262, + "theoretical_loss": 3.7226290692421165, + "tokens_seen": 814284800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804413239719158, + "loss": 2.5049, + "theoretical_loss": 3.722599465501138, + "tokens_seen": 814350336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038043129388164496, + "loss": 2.7759, + "theoretical_loss": 3.722569864809479, + "tokens_seen": 814415872 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038042126379137414, + "loss": 2.6387, + "theoretical_loss": 3.722540267166579, + "tokens_seen": 814481408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804112337011033, + "loss": 2.712, + "theoretical_loss": 3.7225106725718797, + "tokens_seen": 814546944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804012036108325, + "loss": 2.7145, + "theoretical_loss": 3.722481081024821, + "tokens_seen": 814612480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803911735205617, + "loss": 2.8807, + "theoretical_loss": 3.7224514925248444, + "tokens_seen": 814678016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803811434302909, + "loss": 2.6498, + "theoretical_loss": 3.7224219070713915, + "tokens_seen": 814743552 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038037111334002005, + "loss": 2.8192, + "theoretical_loss": 3.722392324663903, + "tokens_seen": 814809088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803610832497493, + "loss": 2.6205, + "theoretical_loss": 3.7223627453018207, + "tokens_seen": 814874624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803510531594784, + "loss": 3.0639, + "theoretical_loss": 3.7223331689845853, + "tokens_seen": 814940160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038034102306920765, + "loss": 2.5076, + "theoretical_loss": 3.7223035957116393, + "tokens_seen": 815005696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038033099297893683, + "loss": 2.4231, + "theoretical_loss": 3.722274025482425, + "tokens_seen": 815071232 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380320962888666, + "loss": 2.5898, + "theoretical_loss": 3.722244458296384, + "tokens_seen": 815136768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803109327983952, + "loss": 2.756, + "theoretical_loss": 3.722214894152958, + "tokens_seen": 815202304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038030090270812437, + "loss": 2.8386, + "theoretical_loss": 3.72218533305159, + "tokens_seen": 815267840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038029087261785355, + "loss": 2.6173, + "theoretical_loss": 3.7221557749917222, + "tokens_seen": 815333376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802808425275828, + "loss": 2.808, + "theoretical_loss": 3.722126219972797, + "tokens_seen": 815398912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802708124373119, + "loss": 2.7507, + "theoretical_loss": 3.7220966679942578, + "tokens_seen": 815464448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038026078234704115, + "loss": 2.5634, + "theoretical_loss": 3.722067119055547, + "tokens_seen": 815529984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038025075225677033, + "loss": 2.7806, + "theoretical_loss": 3.7220375731561086, + "tokens_seen": 815595520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802407221664995, + "loss": 2.6871, + "theoretical_loss": 3.722008030295384, + "tokens_seen": 815661056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802306920762287, + "loss": 2.5132, + "theoretical_loss": 3.7219784904728184, + "tokens_seen": 815726592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802206619859579, + "loss": 2.883, + "theoretical_loss": 3.7219489536878543, + "tokens_seen": 815792128 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038021063189568705, + "loss": 2.8185, + "theoretical_loss": 3.7219194199399355, + "tokens_seen": 815857664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 457884, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9456984996795654, + "objective/train/theoretical_loss": 3.721889889228506, + "objective/train/tokens_used": 836383200, + "theoretical_loss": 3.721889889228506, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802006018054163, + "loss": 2.5694, + "theoretical_loss": 3.721889889228506, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801905717151454, + "loss": 2.4299, + "theoretical_loss": 3.7218603615530093, + "tokens_seen": 815988736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038018054162487465, + "loss": 2.7284, + "theoretical_loss": 3.7218308369128903, + "tokens_seen": 816054272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801705115346038, + "loss": 2.5405, + "theoretical_loss": 3.7218013153075926, + "tokens_seen": 816119808 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380160481444333, + "loss": 2.6405, + "theoretical_loss": 3.7217717967365607, + "tokens_seen": 816185344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801504513540622, + "loss": 2.5513, + "theoretical_loss": 3.7217422811992393, + "tokens_seen": 816250880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801404212637914, + "loss": 2.761, + "theoretical_loss": 3.721712768695073, + "tokens_seen": 816316416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038013039117352056, + "loss": 2.5803, + "theoretical_loss": 3.7216832592235067, + "tokens_seen": 816381952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038012036108324974, + "loss": 2.6744, + "theoretical_loss": 3.7216537527839852, + "tokens_seen": 816447488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801103309929789, + "loss": 2.6138, + "theoretical_loss": 3.7216242493759535, + "tokens_seen": 816513024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038010030090270816, + "loss": 2.4472, + "theoretical_loss": 3.7215947489988572, + "tokens_seen": 816578560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800902708124373, + "loss": 2.4756, + "theoretical_loss": 3.721565251652142, + "tokens_seen": 816644096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800802407221665, + "loss": 2.5205, + "theoretical_loss": 3.7215357573352525, + "tokens_seen": 816709632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800702106318957, + "loss": 2.7038, + "theoretical_loss": 3.7215062660476352, + "tokens_seen": 816775168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800601805416249, + "loss": 2.499, + "theoretical_loss": 3.7214767777887356, + "tokens_seen": 816840704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038005015045135406, + "loss": 2.6442, + "theoretical_loss": 3.7214472925579996, + "tokens_seen": 816906240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038004012036108324, + "loss": 2.792, + "theoretical_loss": 3.7214178103548736, + "tokens_seen": 816971776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800300902708124, + "loss": 2.8231, + "theoretical_loss": 3.7213883311788036, + "tokens_seen": 817037312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038002006018054166, + "loss": 2.6325, + "theoretical_loss": 3.721358855029237, + "tokens_seen": 817102848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800100300902708, + "loss": 2.4452, + "theoretical_loss": 3.721329381905619, + "tokens_seen": 817168384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038, + "loss": 2.7884, + "theoretical_loss": 3.721299911807397, + "tokens_seen": 817233920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037998996990972915, + "loss": 2.6552, + "theoretical_loss": 3.721270444734018, + "tokens_seen": 817299456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799799398194584, + "loss": 2.8386, + "theoretical_loss": 3.7212409806849287, + "tokens_seen": 817364992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037996990972918756, + "loss": 2.5461, + "theoretical_loss": 3.7212115196595765, + "tokens_seen": 817430528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037995987963891675, + "loss": 2.6774, + "theoretical_loss": 3.7211820616574087, + "tokens_seen": 817496064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 458869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.828730344772339, + "objective/train/theoretical_loss": 3.7211526066778724, + "objective/train/tokens_used": 838021600, + "theoretical_loss": 3.7211526066778724, + "tokens_seen": 817561600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799498495486459, + "loss": 2.8248, + "theoretical_loss": 3.7211526066778724, + "tokens_seen": 817561600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037993981945837516, + "loss": 2.6308, + "theoretical_loss": 3.7211231547204155, + "tokens_seen": 817627136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799297893681043, + "loss": 2.7227, + "theoretical_loss": 3.7210937057844857, + "tokens_seen": 817692672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799197592778335, + "loss": 2.4619, + "theoretical_loss": 3.7210642598695305, + "tokens_seen": 817758208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037990972918756265, + "loss": 2.5265, + "theoretical_loss": 3.7210348169749987, + "tokens_seen": 817823744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798996990972919, + "loss": 2.7976, + "theoretical_loss": 3.721005377100338, + "tokens_seen": 817889280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037988966900702107, + "loss": 2.5068, + "theoretical_loss": 3.7209759402449967, + "tokens_seen": 817954816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037987963891675025, + "loss": 2.94, + "theoretical_loss": 3.7209465064084233, + "tokens_seen": 818020352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798696088264795, + "loss": 2.5955, + "theoretical_loss": 3.7209170755900667, + "tokens_seen": 818085888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798595787362086, + "loss": 2.8317, + "theoretical_loss": 3.7208876477893758, + "tokens_seen": 818151424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037984954864593785, + "loss": 2.6954, + "theoretical_loss": 3.7208582230057985, + "tokens_seen": 818216960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037983951855566703, + "loss": 2.8328, + "theoretical_loss": 3.7208288012387847, + "tokens_seen": 818282496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798294884653962, + "loss": 2.5533, + "theoretical_loss": 3.720799382487783, + "tokens_seen": 818348032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798194583751254, + "loss": 2.7027, + "theoretical_loss": 3.7207699667522434, + "tokens_seen": 818413568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037980942828485457, + "loss": 2.4817, + "theoretical_loss": 3.7207405540316154, + "tokens_seen": 818479104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037979939819458375, + "loss": 2.6257, + "theoretical_loss": 3.7207111443253478, + "tokens_seen": 818544640 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379789368104313, + "loss": 2.8737, + "theoretical_loss": 3.7206817376328907, + "tokens_seen": 818610176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797793380140421, + "loss": 2.6199, + "theoretical_loss": 3.7206523339536943, + "tokens_seen": 818675712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037976930792377135, + "loss": 2.5395, + "theoretical_loss": 3.720622933287209, + "tokens_seen": 818741248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037975927783350053, + "loss": 2.6246, + "theoretical_loss": 3.7205935356328848, + "tokens_seen": 818806784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797492477432297, + "loss": 2.797, + "theoretical_loss": 3.720564140990171, + "tokens_seen": 818872320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797392176529589, + "loss": 2.7855, + "theoretical_loss": 3.720534749358519, + "tokens_seen": 818937856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797291875626881, + "loss": 2.7592, + "theoretical_loss": 3.7205053607373797, + "tokens_seen": 819003392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037971915747241725, + "loss": 2.515, + "theoretical_loss": 3.720475975126203, + "tokens_seen": 819068928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797091273821465, + "loss": 2.4283, + "theoretical_loss": 3.72044659252444, + "tokens_seen": 819134464 + }, + { + "debugging/Self-BLEU-5": 0.4343450632679149, + "debugging/distinct-1-grams": 0.7759737985147822, + "debugging/distinct-2-grams": 0.9682119066950529, + "debugging/entropy-1-grams": 5.4098389736484584, + "debugging/entropy-2-grams": 6.131371694974163, + "debugging/length": 546.8333333333334, + "debugging/num_segments": 6, + "epoch": 0.25, + "objective/train/docs_used": 459733, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.788059711456299, + "objective/train/theoretical_loss": 3.720417212931543, + "objective/train/tokens_used": 839660000, + "theoretical_loss": 3.720417212931543, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796990972918756, + "loss": 2.7732, + "theoretical_loss": 3.720417212931543, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037968906720160485, + "loss": 2.6896, + "theoretical_loss": 3.720387836346962, + "tokens_seen": 819265536 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379679037111334, + "loss": 2.5718, + "theoretical_loss": 3.7203584627701485, + "tokens_seen": 819331072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796690070210632, + "loss": 2.7729, + "theoretical_loss": 3.7203290922005543, + "tokens_seen": 819396608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796589769307924, + "loss": 2.7949, + "theoretical_loss": 3.7202997246376306, + "tokens_seen": 819462144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796489468405216, + "loss": 2.7875, + "theoretical_loss": 3.72027036008083, + "tokens_seen": 819527680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037963891675025076, + "loss": 2.5432, + "theoretical_loss": 3.7202409985296034, + "tokens_seen": 819593216 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037962888665997994, + "loss": 2.8756, + "theoretical_loss": 3.7202116399834035, + "tokens_seen": 819658752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796188565697091, + "loss": 2.6533, + "theoretical_loss": 3.720182284441682, + "tokens_seen": 819724288 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037960882647943836, + "loss": 2.8107, + "theoretical_loss": 3.7201529319038924, + "tokens_seen": 819789824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795987963891675, + "loss": 2.8196, + "theoretical_loss": 3.720123582369486, + "tokens_seen": 819855360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795887662988967, + "loss": 2.8017, + "theoretical_loss": 3.720094235837916, + "tokens_seen": 819920896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795787362086259, + "loss": 2.7922, + "theoretical_loss": 3.7200648923086357, + "tokens_seen": 819986432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795687061183551, + "loss": 2.5626, + "theoretical_loss": 3.7200355517810966, + "tokens_seen": 820051968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037955867602808426, + "loss": 2.4739, + "theoretical_loss": 3.7200062142547528, + "tokens_seen": 820117504 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037954864593781344, + "loss": 2.7152, + "theoretical_loss": 3.7199768797290576, + "tokens_seen": 820183040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795386158475426, + "loss": 2.8596, + "theoretical_loss": 3.7199475482034643, + "tokens_seen": 820248576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037952858575727186, + "loss": 2.817, + "theoretical_loss": 3.7199182196774254, + "tokens_seen": 820314112 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379518555667001, + "loss": 2.5966, + "theoretical_loss": 3.719888894150396, + "tokens_seen": 820379648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795085255767302, + "loss": 2.8667, + "theoretical_loss": 3.7198595716218295, + "tokens_seen": 820445184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037949849548645935, + "loss": 2.5299, + "theoretical_loss": 3.719830252091179, + "tokens_seen": 820510720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794884653961886, + "loss": 2.4508, + "theoretical_loss": 3.7198009355578994, + "tokens_seen": 820576256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037947843530591776, + "loss": 2.7271, + "theoretical_loss": 3.7197716220214443, + "tokens_seen": 820641792 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037946840521564695, + "loss": 2.6313, + "theoretical_loss": 3.7197423114812693, + "tokens_seen": 820707328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794583751253761, + "loss": 2.8888, + "theoretical_loss": 3.7197130039368274, + "tokens_seen": 820772864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 460372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5753989219665527, + "objective/train/theoretical_loss": 3.7196836993875744, + "objective/train/tokens_used": 841298400, + "theoretical_loss": 3.7196836993875744, + "tokens_seen": 820838400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037944834503510536, + "loss": 2.6251, + "theoretical_loss": 3.7196836993875744, + "tokens_seen": 820838400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794383149448345, + "loss": 2.6649, + "theoretical_loss": 3.719654397832964, + "tokens_seen": 820903936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794282848545637, + "loss": 2.6335, + "theoretical_loss": 3.7196250992724527, + "tokens_seen": 820969472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037941825476429285, + "loss": 2.7872, + "theoretical_loss": 3.719595803705494, + "tokens_seen": 821035008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794082246740221, + "loss": 2.7039, + "theoretical_loss": 3.7195665111315437, + "tokens_seen": 821100544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037939819458375127, + "loss": 2.5931, + "theoretical_loss": 3.7195372215500573, + "tokens_seen": 821166080 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037938816449348045, + "loss": 2.4618, + "theoretical_loss": 3.7195079349604905, + "tokens_seen": 821231616 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037937813440320963, + "loss": 2.5012, + "theoretical_loss": 3.719478651362299, + "tokens_seen": 821297152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793681043129388, + "loss": 2.6347, + "theoretical_loss": 3.719449370754938, + "tokens_seen": 821362688 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379358074222668, + "loss": 2.5941, + "theoretical_loss": 3.7194200931378636, + "tokens_seen": 821428224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037934804413239723, + "loss": 2.6462, + "theoretical_loss": 3.719390818510532, + "tokens_seen": 821493760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037933801404212635, + "loss": 2.7512, + "theoretical_loss": 3.7193615468724, + "tokens_seen": 821559296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793279839518556, + "loss": 2.7186, + "theoretical_loss": 3.7193322782229226, + "tokens_seen": 821624832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793179538615847, + "loss": 2.7558, + "theoretical_loss": 3.719303012561557, + "tokens_seen": 821690368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037930792377131395, + "loss": 2.676, + "theoretical_loss": 3.719273749887761, + "tokens_seen": 821755904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037929789368104313, + "loss": 2.5643, + "theoretical_loss": 3.71924449020099, + "tokens_seen": 821821440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792878635907723, + "loss": 2.5705, + "theoretical_loss": 3.7192152335007007, + "tokens_seen": 821886976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792778335005015, + "loss": 2.8495, + "theoretical_loss": 3.719185979786351, + "tokens_seen": 821952512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037926780341023073, + "loss": 2.8069, + "theoretical_loss": 3.719156729057398, + "tokens_seen": 822018048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037925777331995986, + "loss": 2.6097, + "theoretical_loss": 3.719127481313299, + "tokens_seen": 822083584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792477432296891, + "loss": 2.5657, + "theoretical_loss": 3.719098236553511, + "tokens_seen": 822149120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792377131394182, + "loss": 2.7598, + "theoretical_loss": 3.7190689947774924, + "tokens_seen": 822214656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037922768304914746, + "loss": 2.8197, + "theoretical_loss": 3.7190397559847006, + "tokens_seen": 822280192 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037921765295887664, + "loss": 2.7267, + "theoretical_loss": 3.7190105201745935, + "tokens_seen": 822345728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792076228686058, + "loss": 2.8076, + "theoretical_loss": 3.7189812873466295, + "tokens_seen": 822411264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 461421, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8889474868774414, + "objective/train/theoretical_loss": 3.7189520575002666, + "objective/train/tokens_used": 842936800, + "theoretical_loss": 3.7189520575002666, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379197592778335, + "loss": 2.8099, + "theoretical_loss": 3.7189520575002666, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791875626880642, + "loss": 2.6691, + "theoretical_loss": 3.7189228306349627, + "tokens_seen": 822542336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037917753259779336, + "loss": 2.6943, + "theoretical_loss": 3.7188936067501768, + "tokens_seen": 822607872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791675025075226, + "loss": 2.8235, + "theoretical_loss": 3.718864385845367, + "tokens_seen": 822673408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791574724172517, + "loss": 2.7914, + "theoretical_loss": 3.7188351679199934, + "tokens_seen": 822738944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037914744232698096, + "loss": 2.7442, + "theoretical_loss": 3.7188059529735136, + "tokens_seen": 822804480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791374122367101, + "loss": 2.7754, + "theoretical_loss": 3.7187767410053874, + "tokens_seen": 822870016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791273821464393, + "loss": 2.5717, + "theoretical_loss": 3.7187475320150734, + "tokens_seen": 822935552 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037911735205616856, + "loss": 2.7173, + "theoretical_loss": 3.7187183260020307, + "tokens_seen": 823001088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791073219658977, + "loss": 2.8607, + "theoretical_loss": 3.7186891229657197, + "tokens_seen": 823066624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790972918756269, + "loss": 2.7677, + "theoretical_loss": 3.7186599229056, + "tokens_seen": 823132160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790872617853561, + "loss": 2.8683, + "theoretical_loss": 3.718630725821131, + "tokens_seen": 823197696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790772316950853, + "loss": 2.6781, + "theoretical_loss": 3.718601531711772, + "tokens_seen": 823263232 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037906720160481446, + "loss": 2.7821, + "theoretical_loss": 3.718572340576984, + "tokens_seen": 823328768 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037905717151454364, + "loss": 2.2804, + "theoretical_loss": 3.7185431524162267, + "tokens_seen": 823394304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790471414242728, + "loss": 2.5982, + "theoretical_loss": 3.7185139672289607, + "tokens_seen": 823459840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037903711133400206, + "loss": 2.7199, + "theoretical_loss": 3.718484785014646, + "tokens_seen": 823525376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790270812437312, + "loss": 2.814, + "theoretical_loss": 3.7184556057727436, + "tokens_seen": 823590912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790170511534604, + "loss": 2.4827, + "theoretical_loss": 3.7184264295027143, + "tokens_seen": 823656448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037900702106318955, + "loss": 2.7661, + "theoretical_loss": 3.718397256204019, + "tokens_seen": 823721984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789969909729188, + "loss": 2.8633, + "theoretical_loss": 3.7183680858761186, + "tokens_seen": 823787520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037898696088264796, + "loss": 2.8532, + "theoretical_loss": 3.718338918518474, + "tokens_seen": 823853056 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037897693079237715, + "loss": 2.5904, + "theoretical_loss": 3.718309754130547, + "tokens_seen": 823918592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789669007021063, + "loss": 2.6954, + "theoretical_loss": 3.718280592711799, + "tokens_seen": 823984128 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037895687061183556, + "loss": 2.9142, + "theoretical_loss": 3.7182514342616906, + "tokens_seen": 824049664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 462012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.056985855102539, + "objective/train/theoretical_loss": 3.7182222787796846, + "objective/train/tokens_used": 844575200, + "theoretical_loss": 3.7182222787796846, + "tokens_seen": 824115200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789468405215647, + "loss": 2.7643, + "theoretical_loss": 3.7182222787796846, + "tokens_seen": 824115200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789368104312939, + "loss": 2.7109, + "theoretical_loss": 3.718193126265243, + "tokens_seen": 824180736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037892678034102305, + "loss": 2.6935, + "theoretical_loss": 3.718163976717828, + "tokens_seen": 824246272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789167502507523, + "loss": 2.5794, + "theoretical_loss": 3.7181348301369006, + "tokens_seen": 824311808 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037890672016048147, + "loss": 2.6364, + "theoretical_loss": 3.7181056865219233, + "tokens_seen": 824377344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037889669007021065, + "loss": 2.678, + "theoretical_loss": 3.7180765458723597, + "tokens_seen": 824442880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037888665997993983, + "loss": 2.6142, + "theoretical_loss": 3.718047408187671, + "tokens_seen": 824508416 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378876629889669, + "loss": 2.6126, + "theoretical_loss": 3.7180182734673206, + "tokens_seen": 824573952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788665997993982, + "loss": 2.3591, + "theoretical_loss": 3.7179891417107718, + "tokens_seen": 824639488 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037885656970912743, + "loss": 2.6444, + "theoretical_loss": 3.717960012917487, + "tokens_seen": 824705024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037884653961885655, + "loss": 2.842, + "theoretical_loss": 3.717930887086929, + "tokens_seen": 824770560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788365095285858, + "loss": 2.8211, + "theoretical_loss": 3.7179017642185617, + "tokens_seen": 824836096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788264794383149, + "loss": 2.7662, + "theoretical_loss": 3.717872644311848, + "tokens_seen": 824901632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037881644934804415, + "loss": 2.6669, + "theoretical_loss": 3.7178435273662522, + "tokens_seen": 824967168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037880641925777333, + "loss": 2.953, + "theoretical_loss": 3.7178144133812374, + "tokens_seen": 825032704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787963891675025, + "loss": 2.6334, + "theoretical_loss": 3.7177853023562673, + "tokens_seen": 825098240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787863590772317, + "loss": 2.847, + "theoretical_loss": 3.7177561942908066, + "tokens_seen": 825163776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037877632898696093, + "loss": 2.6949, + "theoretical_loss": 3.717727089184319, + "tokens_seen": 825229312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037876629889669006, + "loss": 2.6026, + "theoretical_loss": 3.717697987036269, + "tokens_seen": 825294848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787562688064193, + "loss": 2.9019, + "theoretical_loss": 3.71766888784612, + "tokens_seen": 825360384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787462387161484, + "loss": 2.733, + "theoretical_loss": 3.7176397916133377, + "tokens_seen": 825425920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037873620862587766, + "loss": 2.5797, + "theoretical_loss": 3.7176106983373858, + "tokens_seen": 825491456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037872617853560684, + "loss": 2.6358, + "theoretical_loss": 3.7175816080177304, + "tokens_seen": 825556992 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378716148445336, + "loss": 2.93, + "theoretical_loss": 3.717552520653835, + "tokens_seen": 825622528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787061183550652, + "loss": 2.7954, + "theoretical_loss": 3.717523436245166, + "tokens_seen": 825688064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 463076, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.083082914352417, + "objective/train/theoretical_loss": 3.717494354791188, + "objective/train/tokens_used": 846213600, + "theoretical_loss": 3.717494354791188, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786960882647944, + "loss": 3.0909, + "theoretical_loss": 3.717494354791188, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037868605817452356, + "loss": 3.0866, + "theoretical_loss": 3.7174652762913656, + "tokens_seen": 825819136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786760280842528, + "loss": 2.7781, + "theoretical_loss": 3.7174362007451656, + "tokens_seen": 825884672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786659979939819, + "loss": 2.5382, + "theoretical_loss": 3.717407128152053, + "tokens_seen": 825950208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037865596790371116, + "loss": 2.5681, + "theoretical_loss": 3.7173780585114944, + "tokens_seen": 826015744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786459378134403, + "loss": 2.6316, + "theoretical_loss": 3.7173489918229548, + "tokens_seen": 826081280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786359077231695, + "loss": 2.6745, + "theoretical_loss": 3.7173199280859, + "tokens_seen": 826146816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786258776328987, + "loss": 2.7971, + "theoretical_loss": 3.717290867299797, + "tokens_seen": 826212352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786158475426279, + "loss": 2.6546, + "theoretical_loss": 3.717261809464112, + "tokens_seen": 826277888 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037860581745235706, + "loss": 2.5943, + "theoretical_loss": 3.717232754578311, + "tokens_seen": 826343424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785957873620863, + "loss": 2.8337, + "theoretical_loss": 3.717203702641861, + "tokens_seen": 826408960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785857572718154, + "loss": 2.8984, + "theoretical_loss": 3.717174653654229, + "tokens_seen": 826474496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037857572718154466, + "loss": 2.8819, + "theoretical_loss": 3.7171456076148814, + "tokens_seen": 826540032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785656970912738, + "loss": 2.6567, + "theoretical_loss": 3.7171165645232858, + "tokens_seen": 826605568 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378555667001003, + "loss": 2.8761, + "theoretical_loss": 3.7170875243789085, + "tokens_seen": 826671104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785456369107322, + "loss": 2.8024, + "theoretical_loss": 3.717058487181218, + "tokens_seen": 826736640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785356068204614, + "loss": 2.7838, + "theoretical_loss": 3.7170294529296806, + "tokens_seen": 826802176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037852557673019057, + "loss": 2.7053, + "theoretical_loss": 3.7170004216237644, + "tokens_seen": 826867712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037851554663991975, + "loss": 2.8404, + "theoretical_loss": 3.716971393262937, + "tokens_seen": 826933248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037850551654964893, + "loss": 2.7194, + "theoretical_loss": 3.7169423678466664, + "tokens_seen": 826998784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037849548645937816, + "loss": 2.8596, + "theoretical_loss": 3.7169133453744205, + "tokens_seen": 827064320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784854563691073, + "loss": 2.8185, + "theoretical_loss": 3.716884325845668, + "tokens_seen": 827129856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784754262788365, + "loss": 2.5357, + "theoretical_loss": 3.716855309259877, + "tokens_seen": 827195392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037846539618856565, + "loss": 2.6472, + "theoretical_loss": 3.7168262956165146, + "tokens_seen": 827260928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784553660982949, + "loss": 2.6843, + "theoretical_loss": 3.7167972849150512, + "tokens_seen": 827326464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 464152, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.622303009033203, + "objective/train/theoretical_loss": 3.716768277154954, + "objective/train/tokens_used": 847852000, + "theoretical_loss": 3.716768277154954, + "tokens_seen": 827392000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037844533600802407, + "loss": 2.6362, + "theoretical_loss": 3.716768277154954, + "tokens_seen": 827392000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037843530591775325, + "loss": 2.7811, + "theoretical_loss": 3.716739272335693, + "tokens_seen": 827457536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037842527582748243, + "loss": 2.7028, + "theoretical_loss": 3.7167102704567365, + "tokens_seen": 827523072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037841524573721167, + "loss": 2.8274, + "theoretical_loss": 3.7166812715175537, + "tokens_seen": 827588608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784052156469408, + "loss": 2.7487, + "theoretical_loss": 3.716652275517614, + "tokens_seen": 827654144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037839518555667003, + "loss": 2.7647, + "theoretical_loss": 3.716623282456387, + "tokens_seen": 827719680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037838515546639916, + "loss": 2.9639, + "theoretical_loss": 3.7165942923333417, + "tokens_seen": 827785216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783751253761284, + "loss": 2.8212, + "theoretical_loss": 3.716565305147948, + "tokens_seen": 827850752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037836509528585763, + "loss": 2.4917, + "theoretical_loss": 3.7165363208996753, + "tokens_seen": 827916288 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037835506519558675, + "loss": 2.9519, + "theoretical_loss": 3.716507339587995, + "tokens_seen": 827981824 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378345035105316, + "loss": 2.9141, + "theoretical_loss": 3.716478361212375, + "tokens_seen": 828047360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783350050150451, + "loss": 2.7782, + "theoretical_loss": 3.716449385772288, + "tokens_seen": 828112896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037832497492477435, + "loss": 2.6236, + "theoretical_loss": 3.716420413267201, + "tokens_seen": 828178432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037831494483450353, + "loss": 2.834, + "theoretical_loss": 3.7163914436965877, + "tokens_seen": 828243968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783049147442327, + "loss": 2.597, + "theoretical_loss": 3.7163624770599175, + "tokens_seen": 828309504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782948846539619, + "loss": 2.7529, + "theoretical_loss": 3.716333513356661, + "tokens_seen": 828375040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037828485456369113, + "loss": 2.3636, + "theoretical_loss": 3.7163045525862897, + "tokens_seen": 828440576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037827482447342026, + "loss": 2.833, + "theoretical_loss": 3.7162755947482733, + "tokens_seen": 828506112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782647943831495, + "loss": 2.7211, + "theoretical_loss": 3.716246639842084, + "tokens_seen": 828571648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782547642928786, + "loss": 2.7162, + "theoretical_loss": 3.7162176878671938, + "tokens_seen": 828637184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037824473420260786, + "loss": 2.7256, + "theoretical_loss": 3.7161887388230728, + "tokens_seen": 828702720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037823470411233704, + "loss": 2.9198, + "theoretical_loss": 3.7161597927091927, + "tokens_seen": 828768256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782246740220662, + "loss": 2.8702, + "theoretical_loss": 3.716130849525026, + "tokens_seen": 828833792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782146439317954, + "loss": 2.7121, + "theoretical_loss": 3.7161019092700442, + "tokens_seen": 828899328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782046138415246, + "loss": 2.5721, + "theoretical_loss": 3.716072971943719, + "tokens_seen": 828964864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 464802, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.640942096710205, + "objective/train/theoretical_loss": 3.716044037545523, + "objective/train/tokens_used": 849490400, + "theoretical_loss": 3.716044037545523, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037819458375125376, + "loss": 2.6406, + "theoretical_loss": 3.716044037545523, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378184553660983, + "loss": 2.7612, + "theoretical_loss": 3.716015106074928, + "tokens_seen": 829095936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781745235707121, + "loss": 2.5001, + "theoretical_loss": 3.715986177531407, + "tokens_seen": 829161472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037816449348044136, + "loss": 2.8508, + "theoretical_loss": 3.715957251914432, + "tokens_seen": 829227008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781544633901705, + "loss": 2.9323, + "theoretical_loss": 3.715928329223476, + "tokens_seen": 829292544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781444332998997, + "loss": 2.7472, + "theoretical_loss": 3.715899409458012, + "tokens_seen": 829358080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781344032096289, + "loss": 2.8221, + "theoretical_loss": 3.7158704926175123, + "tokens_seen": 829423616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781243731193581, + "loss": 2.7649, + "theoretical_loss": 3.7158415787014505, + "tokens_seen": 829489152 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037811434302908726, + "loss": 2.7626, + "theoretical_loss": 3.7158126677092995, + "tokens_seen": 829554688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781043129388165, + "loss": 2.777, + "theoretical_loss": 3.7157837596405328, + "tokens_seen": 829620224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780942828485456, + "loss": 2.8599, + "theoretical_loss": 3.7157548544946244, + "tokens_seen": 829685760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037808425275827486, + "loss": 2.7063, + "theoretical_loss": 3.7157259522710473, + "tokens_seen": 829751296 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378074222668004, + "loss": 2.7576, + "theoretical_loss": 3.7156970529692757, + "tokens_seen": 829816832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780641925777332, + "loss": 2.6135, + "theoretical_loss": 3.7156681565887824, + "tokens_seen": 829882368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780541624874624, + "loss": 2.6237, + "theoretical_loss": 3.715639263129043, + "tokens_seen": 829947904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780441323971916, + "loss": 2.6227, + "theoretical_loss": 3.7156103725895306, + "tokens_seen": 830013440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037803410230692077, + "loss": 2.5027, + "theoretical_loss": 3.71558148496972, + "tokens_seen": 830078976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037802407221664995, + "loss": 2.6977, + "theoretical_loss": 3.7155526002690857, + "tokens_seen": 830144512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037801404212637913, + "loss": 2.5946, + "theoretical_loss": 3.715523718487102, + "tokens_seen": 830210048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037800401203610836, + "loss": 2.7186, + "theoretical_loss": 3.7154948396232443, + "tokens_seen": 830275584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779939819458375, + "loss": 2.7893, + "theoretical_loss": 3.7154659636769862, + "tokens_seen": 830341120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037798395185556673, + "loss": 2.7367, + "theoretical_loss": 3.7154370906478036, + "tokens_seen": 830406656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037797392176529585, + "loss": 2.6869, + "theoretical_loss": 3.7154082205351715, + "tokens_seen": 830472192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779638916750251, + "loss": 2.8705, + "theoretical_loss": 3.715379353338565, + "tokens_seen": 830537728 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037795386158475427, + "loss": 2.7076, + "theoretical_loss": 3.7153504890574602, + "tokens_seen": 830603264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 465950, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4607279300689697, + "objective/train/theoretical_loss": 3.7153216276913312, + "objective/train/tokens_used": 851128800, + "theoretical_loss": 3.7153216276913312, + "tokens_seen": 830668800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037794383149448345, + "loss": 2.4907, + "theoretical_loss": 3.7153216276913312, + "tokens_seen": 830668800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037793380140421263, + "loss": 2.7686, + "theoretical_loss": 3.7152927692396553, + "tokens_seen": 830734336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037792377131394187, + "loss": 2.9923, + "theoretical_loss": 3.7152639137019072, + "tokens_seen": 830799872 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377913741223671, + "loss": 2.7894, + "theoretical_loss": 3.7152350610775633, + "tokens_seen": 830865408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037790371113340023, + "loss": 2.3556, + "theoretical_loss": 3.7152062113660995, + "tokens_seen": 830930944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037789368104312936, + "loss": 2.7755, + "theoretical_loss": 3.7151773645669923, + "tokens_seen": 830996480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778836509528586, + "loss": 2.7287, + "theoretical_loss": 3.7151485206797177, + "tokens_seen": 831062016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778736208625878, + "loss": 2.8398, + "theoretical_loss": 3.7151196797037525, + "tokens_seen": 831127552 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037786359077231695, + "loss": 2.686, + "theoretical_loss": 3.715090841638573, + "tokens_seen": 831193088 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037785356068204614, + "loss": 2.6514, + "theoretical_loss": 3.715062006483656, + "tokens_seen": 831258624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778435305917753, + "loss": 2.8311, + "theoretical_loss": 3.7150331742384783, + "tokens_seen": 831324160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778335005015045, + "loss": 2.819, + "theoretical_loss": 3.7150043449025176, + "tokens_seen": 831389696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037782347041123373, + "loss": 2.8208, + "theoretical_loss": 3.71497551847525, + "tokens_seen": 831455232 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037781344032096286, + "loss": 2.6147, + "theoretical_loss": 3.7149466949561547, + "tokens_seen": 831520768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778034102306921, + "loss": 2.6827, + "theoretical_loss": 3.7149178743447067, + "tokens_seen": 831586304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777933801404213, + "loss": 2.7768, + "theoretical_loss": 3.714889056640385, + "tokens_seen": 831651840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037778335005015046, + "loss": 2.4861, + "theoretical_loss": 3.7148602418426675, + "tokens_seen": 831717376 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037777331995987964, + "loss": 2.6438, + "theoretical_loss": 3.7148314299510314, + "tokens_seen": 831782912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777632898696088, + "loss": 2.8952, + "theoretical_loss": 3.7148026209649543, + "tokens_seen": 831848448 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377753259779338, + "loss": 2.7831, + "theoretical_loss": 3.714773814883915, + "tokens_seen": 831913984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037774322968906724, + "loss": 2.751, + "theoretical_loss": 3.714745011707392, + "tokens_seen": 831979520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037773319959879636, + "loss": 2.5419, + "theoretical_loss": 3.714716211434863, + "tokens_seen": 832045056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777231695085256, + "loss": 2.954, + "theoretical_loss": 3.714687414065807, + "tokens_seen": 832110592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777131394182547, + "loss": 2.6081, + "theoretical_loss": 3.7146586195997022, + "tokens_seen": 832176128 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037770310932798396, + "loss": 2.8164, + "theoretical_loss": 3.7146298280360277, + "tokens_seen": 832241664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 466904, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.74540638923645, + "objective/train/theoretical_loss": 3.714601039374263, + "objective/train/tokens_used": 852767200, + "theoretical_loss": 3.714601039374263, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037769307923771314, + "loss": 2.7948, + "theoretical_loss": 3.714601039374263, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776830491474423, + "loss": 2.7842, + "theoretical_loss": 3.714572253613886, + "tokens_seen": 832372736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776730190571715, + "loss": 2.7081, + "theoretical_loss": 3.714543470754376, + "tokens_seen": 832438272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776629889669007, + "loss": 2.7968, + "theoretical_loss": 3.714514690795213, + "tokens_seen": 832503808 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037765295887662987, + "loss": 2.92, + "theoretical_loss": 3.714485913735876, + "tokens_seen": 832569344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776429287863591, + "loss": 2.6305, + "theoretical_loss": 3.7144571395758454, + "tokens_seen": 832634880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037763289869608823, + "loss": 2.8311, + "theoretical_loss": 3.7144283683145995, + "tokens_seen": 832700416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037762286860581746, + "loss": 2.9982, + "theoretical_loss": 3.7143995999516193, + "tokens_seen": 832765952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776128385155467, + "loss": 2.915, + "theoretical_loss": 3.7143708344863846, + "tokens_seen": 832831488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776028084252758, + "loss": 2.6645, + "theoretical_loss": 3.7143420719183755, + "tokens_seen": 832897024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037759277833500506, + "loss": 2.8094, + "theoretical_loss": 3.7143133122470715, + "tokens_seen": 832962560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775827482447342, + "loss": 2.8851, + "theoretical_loss": 3.7142845554719544, + "tokens_seen": 833028096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775727181544634, + "loss": 2.6911, + "theoretical_loss": 3.714255801592503, + "tokens_seen": 833093632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775626880641926, + "loss": 2.7356, + "theoretical_loss": 3.7142270506081996, + "tokens_seen": 833159168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775526579739218, + "loss": 2.9705, + "theoretical_loss": 3.7141983025185246, + "tokens_seen": 833224704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037754262788365097, + "loss": 2.901, + "theoretical_loss": 3.714169557322958, + "tokens_seen": 833290240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037753259779338015, + "loss": 2.9581, + "theoretical_loss": 3.714140815020982, + "tokens_seen": 833355776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037752256770310933, + "loss": 2.7036, + "theoretical_loss": 3.7141120756120767, + "tokens_seen": 833421312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037751253761283857, + "loss": 2.7045, + "theoretical_loss": 3.714083339095725, + "tokens_seen": 833486848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775025075225677, + "loss": 2.8971, + "theoretical_loss": 3.7140546054714068, + "tokens_seen": 833552384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037749247743229693, + "loss": 2.6651, + "theoretical_loss": 3.7140258747386046, + "tokens_seen": 833617920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037748244734202605, + "loss": 2.7434, + "theoretical_loss": 3.7139971468967996, + "tokens_seen": 833683456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774724172517553, + "loss": 2.8242, + "theoretical_loss": 3.713968421945474, + "tokens_seen": 833748992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037746238716148447, + "loss": 2.7712, + "theoretical_loss": 3.7139396998841097, + "tokens_seen": 833814528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037745235707121365, + "loss": 2.8285, + "theoretical_loss": 3.713910980712189, + "tokens_seen": 833880064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 467427, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3416881561279297, + "objective/train/theoretical_loss": 3.713882264429194, + "objective/train/tokens_used": 854405600, + "theoretical_loss": 3.713882264429194, + "tokens_seen": 833945600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037744232698094283, + "loss": 2.7444, + "theoretical_loss": 3.713882264429194, + "tokens_seen": 833945600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037743229689067207, + "loss": 2.7785, + "theoretical_loss": 3.713853551034607, + "tokens_seen": 834011136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774222668004012, + "loss": 2.7879, + "theoretical_loss": 3.713824840527911, + "tokens_seen": 834076672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037741223671013043, + "loss": 2.8718, + "theoretical_loss": 3.7137961329085885, + "tokens_seen": 834142208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037740220661985956, + "loss": 2.618, + "theoretical_loss": 3.713767428176122, + "tokens_seen": 834207744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773921765295888, + "loss": 2.6378, + "theoretical_loss": 3.713738726329994, + "tokens_seen": 834273280 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377382146439318, + "loss": 2.7106, + "theoretical_loss": 3.7137100273696886, + "tokens_seen": 834338816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037737211634904715, + "loss": 2.7905, + "theoretical_loss": 3.713681331294689, + "tokens_seen": 834404352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037736208625877634, + "loss": 2.8341, + "theoretical_loss": 3.7136526381044774, + "tokens_seen": 834469888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773520561685055, + "loss": 3.0653, + "theoretical_loss": 3.7136239477985384, + "tokens_seen": 834535424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773420260782347, + "loss": 2.6993, + "theoretical_loss": 3.713595260376355, + "tokens_seen": 834600960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037733199598796393, + "loss": 2.6912, + "theoretical_loss": 3.713566575837411, + "tokens_seen": 834666496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037732196589769306, + "loss": 2.887, + "theoretical_loss": 3.713537894181191, + "tokens_seen": 834732032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773119358074223, + "loss": 2.8895, + "theoretical_loss": 3.7135092154071776, + "tokens_seen": 834797568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773019057171515, + "loss": 2.737, + "theoretical_loss": 3.7134805395148556, + "tokens_seen": 834863104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037729187562688066, + "loss": 2.5348, + "theoretical_loss": 3.71345186650371, + "tokens_seen": 834928640 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037728184553660984, + "loss": 2.673, + "theoretical_loss": 3.7134231963732245, + "tokens_seen": 834994176 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377271815446339, + "loss": 2.5236, + "theoretical_loss": 3.7133945291228834, + "tokens_seen": 835059712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772617853560682, + "loss": 2.8603, + "theoretical_loss": 3.713365864752172, + "tokens_seen": 835125248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037725175526579744, + "loss": 2.7683, + "theoretical_loss": 3.7133372032605747, + "tokens_seen": 835190784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037724172517552656, + "loss": 2.822, + "theoretical_loss": 3.7133085446475764, + "tokens_seen": 835256320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772316950852558, + "loss": 2.6931, + "theoretical_loss": 3.7132798889126626, + "tokens_seen": 835321856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772216649949849, + "loss": 2.5958, + "theoretical_loss": 3.713251236055317, + "tokens_seen": 835387392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037721163490471416, + "loss": 2.7673, + "theoretical_loss": 3.7132225860750276, + "tokens_seen": 835452928 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037720160481444334, + "loss": 2.8958, + "theoretical_loss": 3.713193938971278, + "tokens_seen": 835518464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 468837, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6179723739624023, + "objective/train/theoretical_loss": 3.7131652947435536, + "objective/train/tokens_used": 856044000, + "theoretical_loss": 3.7131652947435536, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771915747241725, + "loss": 2.513, + "theoretical_loss": 3.7131652947435536, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771815446339017, + "loss": 2.6243, + "theoretical_loss": 3.7131366533913406, + "tokens_seen": 835649536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771715145436309, + "loss": 2.6509, + "theoretical_loss": 3.713108014914125, + "tokens_seen": 835715072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037716148445336007, + "loss": 2.594, + "theoretical_loss": 3.7130793793113934, + "tokens_seen": 835780608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771514543630893, + "loss": 2.697, + "theoretical_loss": 3.7130507465826303, + "tokens_seen": 835846144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037714142427281843, + "loss": 2.6427, + "theoretical_loss": 3.713022116727324, + "tokens_seen": 835911680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037713139418254766, + "loss": 2.9435, + "theoretical_loss": 3.7129934897449592, + "tokens_seen": 835977216 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037712136409227685, + "loss": 2.7768, + "theoretical_loss": 3.7129648656350227, + "tokens_seen": 836042752 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377111334002006, + "loss": 2.7805, + "theoretical_loss": 3.7129362443970013, + "tokens_seen": 836108288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771013039117352, + "loss": 2.5428, + "theoretical_loss": 3.7129076260303826, + "tokens_seen": 836173824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770912738214644, + "loss": 2.8572, + "theoretical_loss": 3.7128790105346523, + "tokens_seen": 836239360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037708124373119357, + "loss": 2.6759, + "theoretical_loss": 3.7128503979092984, + "tokens_seen": 836304896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770712136409228, + "loss": 2.9712, + "theoretical_loss": 3.7128217881538075, + "tokens_seen": 836370432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037706118355065193, + "loss": 2.9666, + "theoretical_loss": 3.712793181267667, + "tokens_seen": 836435968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037705115346038117, + "loss": 2.6751, + "theoretical_loss": 3.7127645772503644, + "tokens_seen": 836501504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770411233701103, + "loss": 2.9037, + "theoretical_loss": 3.7127359761013876, + "tokens_seen": 836567040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037703109327983953, + "loss": 2.8934, + "theoretical_loss": 3.7127073778202235, + "tokens_seen": 836632576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770210631895687, + "loss": 2.7859, + "theoretical_loss": 3.7126787824063605, + "tokens_seen": 836698112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770110330992979, + "loss": 2.5454, + "theoretical_loss": 3.7126501898592865, + "tokens_seen": 836763648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770010030090271, + "loss": 2.836, + "theoretical_loss": 3.71262160017849, + "tokens_seen": 836829184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037699097291875625, + "loss": 2.8422, + "theoretical_loss": 3.7125930133634584, + "tokens_seen": 836894720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037698094282848544, + "loss": 2.8695, + "theoretical_loss": 3.712564429413681, + "tokens_seen": 836960256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037697091273821467, + "loss": 2.9816, + "theoretical_loss": 3.7125358483286455, + "tokens_seen": 837025792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769608826479438, + "loss": 2.7577, + "theoretical_loss": 3.71250727010784, + "tokens_seen": 837091328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037695085255767303, + "loss": 2.8082, + "theoretical_loss": 3.7124786947507546, + "tokens_seen": 837156864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 470129, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7207224369049072, + "objective/train/theoretical_loss": 3.712450122256878, + "objective/train/tokens_used": 857682400, + "theoretical_loss": 3.712450122256878, + "tokens_seen": 837222400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769408224674022, + "loss": 2.7488, + "theoretical_loss": 3.712450122256878, + "tokens_seen": 837222400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769307923771314, + "loss": 2.6886, + "theoretical_loss": 3.712421552625699, + "tokens_seen": 837287936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769207622868606, + "loss": 2.4888, + "theoretical_loss": 3.712392985856706, + "tokens_seen": 837353472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037691073219658976, + "loss": 2.9448, + "theoretical_loss": 3.712364421949389, + "tokens_seen": 837419008 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037690070210631894, + "loss": 2.7908, + "theoretical_loss": 3.712335860903238, + "tokens_seen": 837484544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768906720160482, + "loss": 2.7475, + "theoretical_loss": 3.7123073027177407, + "tokens_seen": 837550080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768806419257773, + "loss": 2.8197, + "theoretical_loss": 3.712278747392389, + "tokens_seen": 837615616 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037687061183550654, + "loss": 2.3862, + "theoretical_loss": 3.712250194926671, + "tokens_seen": 837681152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768605817452357, + "loss": 2.7735, + "theoretical_loss": 3.712221645320077, + "tokens_seen": 837746688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768505516549649, + "loss": 2.7545, + "theoretical_loss": 3.712193098572098, + "tokens_seen": 837812224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037684052156469413, + "loss": 2.5102, + "theoretical_loss": 3.7121645546822233, + "tokens_seen": 837877760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037683049147442326, + "loss": 2.8537, + "theoretical_loss": 3.7121360136499435, + "tokens_seen": 837943296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768204613841525, + "loss": 2.5985, + "theoretical_loss": 3.712107475474749, + "tokens_seen": 838008832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768104312938817, + "loss": 2.97, + "theoretical_loss": 3.7120789401561303, + "tokens_seen": 838074368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037680040120361086, + "loss": 2.7995, + "theoretical_loss": 3.712050407693578, + "tokens_seen": 838139904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037679037111334004, + "loss": 3.0605, + "theoretical_loss": 3.712021878086584, + "tokens_seen": 838205440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767803410230692, + "loss": 2.8332, + "theoretical_loss": 3.7119933513346375, + "tokens_seen": 838270976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767703109327984, + "loss": 2.9444, + "theoretical_loss": 3.7119648274372308, + "tokens_seen": 838336512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037676028084252764, + "loss": 3.0157, + "theoretical_loss": 3.711936306393855, + "tokens_seen": 838402048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037675025075225676, + "loss": 2.6819, + "theoretical_loss": 3.7119077882040017, + "tokens_seen": 838467584 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376740220661986, + "loss": 2.7764, + "theoretical_loss": 3.711879272867162, + "tokens_seen": 838533120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767301905717151, + "loss": 2.7516, + "theoretical_loss": 3.711850760382827, + "tokens_seen": 838598656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037672016048144436, + "loss": 3.0291, + "theoretical_loss": 3.71182225075049, + "tokens_seen": 838664192 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037671013039117354, + "loss": 2.9408, + "theoretical_loss": 3.7117937439696416, + "tokens_seen": 838729728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767001003009027, + "loss": 2.6546, + "theoretical_loss": 3.7117652400397736, + "tokens_seen": 838795264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 470653, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.006452798843384, + "objective/train/theoretical_loss": 3.7117367389603793, + "objective/train/tokens_used": 859320800, + "theoretical_loss": 3.7117367389603793, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766900702106319, + "loss": 2.886, + "theoretical_loss": 3.7117367389603793, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766800401203611, + "loss": 2.6274, + "theoretical_loss": 3.7117082407309505, + "tokens_seen": 838926336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037667001003009027, + "loss": 2.8262, + "theoretical_loss": 3.7116797453509793, + "tokens_seen": 838991872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766599799398195, + "loss": 2.8575, + "theoretical_loss": 3.711651252819959, + "tokens_seen": 839057408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037664994984954863, + "loss": 2.7635, + "theoretical_loss": 3.711622763137381, + "tokens_seen": 839122944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037663991975927786, + "loss": 2.7222, + "theoretical_loss": 3.7115942763027396, + "tokens_seen": 839188480 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037662988966900705, + "loss": 2.79, + "theoretical_loss": 3.7115657923155267, + "tokens_seen": 839254016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766198595787362, + "loss": 2.621, + "theoretical_loss": 3.711537311175236, + "tokens_seen": 839319552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766098294884654, + "loss": 2.7895, + "theoretical_loss": 3.7115088328813597, + "tokens_seen": 839385088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765997993981946, + "loss": 2.7553, + "theoretical_loss": 3.7114803574333926, + "tokens_seen": 839450624 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037658976930792377, + "loss": 2.6999, + "theoretical_loss": 3.711451884830827, + "tokens_seen": 839516160 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376579739217653, + "loss": 2.827, + "theoretical_loss": 3.7114234150731567, + "tokens_seen": 839581696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037656970912738213, + "loss": 2.5286, + "theoretical_loss": 3.711394948159876, + "tokens_seen": 839647232 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037655967903711137, + "loss": 2.7859, + "theoretical_loss": 3.711366484090478, + "tokens_seen": 839712768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765496489468405, + "loss": 2.8513, + "theoretical_loss": 3.7113380228644566, + "tokens_seen": 839778304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037653961885656973, + "loss": 2.6668, + "theoretical_loss": 3.711309564481307, + "tokens_seen": 839843840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765295887662989, + "loss": 2.8288, + "theoretical_loss": 3.711281108940522, + "tokens_seen": 839909376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765195586760281, + "loss": 2.907, + "theoretical_loss": 3.711252656241597, + "tokens_seen": 839974912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765095285857573, + "loss": 2.6672, + "theoretical_loss": 3.711224206384026, + "tokens_seen": 840040448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037649949849548645, + "loss": 2.5705, + "theoretical_loss": 3.711195759367304, + "tokens_seen": 840105984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037648946840521564, + "loss": 2.5694, + "theoretical_loss": 3.7111673151909255, + "tokens_seen": 840171520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037647943831494487, + "loss": 2.9223, + "theoretical_loss": 3.7111388738543853, + "tokens_seen": 840237056 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376469408224674, + "loss": 2.7859, + "theoretical_loss": 3.7111104353571784, + "tokens_seen": 840302592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037645937813440323, + "loss": 2.8041, + "theoretical_loss": 3.7110819996988003, + "tokens_seen": 840368128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764493480441324, + "loss": 2.6228, + "theoretical_loss": 3.711053566878746, + "tokens_seen": 840433664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 471790, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.994243860244751, + "objective/train/theoretical_loss": 3.711025136896511, + "objective/train/tokens_used": 860959200, + "theoretical_loss": 3.711025136896511, + "tokens_seen": 840499200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764393179538616, + "loss": 2.864, + "theoretical_loss": 3.711025136896511, + "tokens_seen": 840499200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764292878635908, + "loss": 2.5993, + "theoretical_loss": 3.7109967097515906, + "tokens_seen": 840564736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037641925777331996, + "loss": 2.894, + "theoretical_loss": 3.7109682854434802, + "tokens_seen": 840630272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037640922768304914, + "loss": 2.7505, + "theoretical_loss": 3.710939863971676, + "tokens_seen": 840695808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763991975927784, + "loss": 2.8284, + "theoretical_loss": 3.7109114453356744, + "tokens_seen": 840761344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763891675025075, + "loss": 2.9597, + "theoretical_loss": 3.710883029534971, + "tokens_seen": 840826880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037637913741223674, + "loss": 2.5393, + "theoretical_loss": 3.710854616569061, + "tokens_seen": 840892416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037636910732196586, + "loss": 2.6253, + "theoretical_loss": 3.710826206437442, + "tokens_seen": 840957952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763590772316951, + "loss": 2.9871, + "theoretical_loss": 3.7107977991396104, + "tokens_seen": 841023488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763490471414243, + "loss": 2.5868, + "theoretical_loss": 3.7107693946750624, + "tokens_seen": 841089024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037633901705115346, + "loss": 2.8583, + "theoretical_loss": 3.710740993043294, + "tokens_seen": 841154560 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037632898696088264, + "loss": 2.642, + "theoretical_loss": 3.7107125942438035, + "tokens_seen": 841220096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763189568706119, + "loss": 3.005, + "theoretical_loss": 3.710684198276086, + "tokens_seen": 841285632 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376308926780341, + "loss": 2.6299, + "theoretical_loss": 3.7106558051396403, + "tokens_seen": 841351168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037629889669007024, + "loss": 2.8208, + "theoretical_loss": 3.7106274148339624, + "tokens_seen": 841416704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037628886659979937, + "loss": 2.6842, + "theoretical_loss": 3.7105990273585503, + "tokens_seen": 841482240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762788365095286, + "loss": 2.8769, + "theoretical_loss": 3.7105706427129013, + "tokens_seen": 841547776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762688064192578, + "loss": 2.8247, + "theoretical_loss": 3.710542260896513, + "tokens_seen": 841613312 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037625877632898696, + "loss": 2.8089, + "theoretical_loss": 3.710513881908883, + "tokens_seen": 841678848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037624874623871615, + "loss": 2.6378, + "theoretical_loss": 3.710485505749509, + "tokens_seen": 841744384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762387161484453, + "loss": 2.6468, + "theoretical_loss": 3.7104571324178894, + "tokens_seen": 841809920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762286860581745, + "loss": 2.7883, + "theoretical_loss": 3.710428761913522, + "tokens_seen": 841875456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037621865596790374, + "loss": 2.6884, + "theoretical_loss": 3.710400394235905, + "tokens_seen": 841940992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037620862587763287, + "loss": 2.7462, + "theoretical_loss": 3.710372029384537, + "tokens_seen": 842006528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761985957873621, + "loss": 2.7431, + "theoretical_loss": 3.7103436673589156, + "tokens_seen": 842072064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 472419, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.929213762283325, + "objective/train/theoretical_loss": 3.710315308158541, + "objective/train/tokens_used": 862597600, + "theoretical_loss": 3.710315308158541, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037618856569709123, + "loss": 2.922, + "theoretical_loss": 3.710315308158541, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037617853560682047, + "loss": 2.7697, + "theoretical_loss": 3.7102869517829102, + "tokens_seen": 842203136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037616850551654965, + "loss": 2.5601, + "theoretical_loss": 3.7102585982315235, + "tokens_seen": 842268672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037615847542627883, + "loss": 2.746, + "theoretical_loss": 3.7102302475038793, + "tokens_seen": 842334208 + }, + { + "epoch": 0.26, + "learning_rate": 0.000376148445336008, + "loss": 2.8692, + "theoretical_loss": 3.7102018995994763, + "tokens_seen": 842399744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037613841524573725, + "loss": 2.585, + "theoretical_loss": 3.710173554517815, + "tokens_seen": 842465280 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037612838515546637, + "loss": 2.4941, + "theoretical_loss": 3.710145212258393, + "tokens_seen": 842530816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761183550651956, + "loss": 2.7491, + "theoretical_loss": 3.710116872820711, + "tokens_seen": 842596352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761083249749248, + "loss": 2.7298, + "theoretical_loss": 3.7100885362042684, + "tokens_seen": 842661888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037609829488465397, + "loss": 3.0135, + "theoretical_loss": 3.710060202408565, + "tokens_seen": 842727424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760882647943832, + "loss": 2.7064, + "theoretical_loss": 3.7100318714331006, + "tokens_seen": 842792960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037607823470411233, + "loss": 2.8233, + "theoretical_loss": 3.710003543277376, + "tokens_seen": 842858496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037606820461384157, + "loss": 2.6252, + "theoretical_loss": 3.7099752179408894, + "tokens_seen": 842924032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760581745235707, + "loss": 2.6879, + "theoretical_loss": 3.709946895423143, + "tokens_seen": 842989568 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037604814443329993, + "loss": 2.7986, + "theoretical_loss": 3.709918575723636, + "tokens_seen": 843055104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760381143430291, + "loss": 2.8934, + "theoretical_loss": 3.7098902588418694, + "tokens_seen": 843120640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760280842527583, + "loss": 2.7243, + "theoretical_loss": 3.7098619447773444, + "tokens_seen": 843186176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760180541624875, + "loss": 2.7749, + "theoretical_loss": 3.7098336335295605, + "tokens_seen": 843251712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037600802407221665, + "loss": 2.7942, + "theoretical_loss": 3.70980532509802, + "tokens_seen": 843317248 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037599799398194584, + "loss": 2.636, + "theoretical_loss": 3.7097770194822224, + "tokens_seen": 843382784 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037598796389167507, + "loss": 2.5375, + "theoretical_loss": 3.70974871668167, + "tokens_seen": 843448320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759779338014042, + "loss": 2.8475, + "theoretical_loss": 3.709720416695864, + "tokens_seen": 843513856 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037596790371113343, + "loss": 2.6591, + "theoretical_loss": 3.709692119524305, + "tokens_seen": 843579392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759578736208626, + "loss": 2.4604, + "theoretical_loss": 3.709663825166496, + "tokens_seen": 843644928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759478435305918, + "loss": 2.8007, + "theoretical_loss": 3.7096355336219373, + "tokens_seen": 843710464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 473620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1842246055603027, + "objective/train/theoretical_loss": 3.709607244890131, + "objective/train/tokens_used": 864236000, + "theoretical_loss": 3.709607244890131, + "tokens_seen": 843776000 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375937813440321, + "loss": 2.5554, + "theoretical_loss": 3.709607244890131, + "tokens_seen": 843776000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037592778335005016, + "loss": 2.854, + "theoretical_loss": 3.70957895897058, + "tokens_seen": 843841536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037591775325977934, + "loss": 2.844, + "theoretical_loss": 3.7095506758627845, + "tokens_seen": 843907072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759077231695086, + "loss": 2.656, + "theoretical_loss": 3.7095223955662484, + "tokens_seen": 843972608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758976930792377, + "loss": 2.69, + "theoretical_loss": 3.709494118080473, + "tokens_seen": 844038144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037588766298896694, + "loss": 2.6778, + "theoretical_loss": 3.7094658434049617, + "tokens_seen": 844103680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037587763289869606, + "loss": 2.9431, + "theoretical_loss": 3.709437571539216, + "tokens_seen": 844169216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758676028084253, + "loss": 2.8346, + "theoretical_loss": 3.709409302482739, + "tokens_seen": 844234752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758575727181545, + "loss": 2.7605, + "theoretical_loss": 3.7093810362350332, + "tokens_seen": 844300288 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037584754262788366, + "loss": 2.5804, + "theoretical_loss": 3.7093527727956026, + "tokens_seen": 844365824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037583751253761284, + "loss": 2.5707, + "theoretical_loss": 3.709324512163949, + "tokens_seen": 844431360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758274824473421, + "loss": 2.9522, + "theoretical_loss": 3.709296254339576, + "tokens_seen": 844496896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758174523570712, + "loss": 2.804, + "theoretical_loss": 3.7092679993219875, + "tokens_seen": 844562432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037580742226680044, + "loss": 2.6918, + "theoretical_loss": 3.709239747110686, + "tokens_seen": 844627968 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037579739217652957, + "loss": 2.7089, + "theoretical_loss": 3.7092114977051756, + "tokens_seen": 844693504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757873620862588, + "loss": 2.8012, + "theoretical_loss": 3.7091832511049603, + "tokens_seen": 844759040 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375777331995988, + "loss": 2.5886, + "theoretical_loss": 3.709155007309543, + "tokens_seen": 844824576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037576730190571716, + "loss": 2.7937, + "theoretical_loss": 3.7091267663184286, + "tokens_seen": 844890112 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037575727181544635, + "loss": 2.6963, + "theoretical_loss": 3.7090985281311206, + "tokens_seen": 844955648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757472417251755, + "loss": 2.7533, + "theoretical_loss": 3.7090702927471235, + "tokens_seen": 845021184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757372116349047, + "loss": 2.7116, + "theoretical_loss": 3.7090420601659417, + "tokens_seen": 845086720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037572718154463394, + "loss": 2.975, + "theoretical_loss": 3.7090138303870788, + "tokens_seen": 845152256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037571715145436307, + "loss": 2.8198, + "theoretical_loss": 3.7089856034100404, + "tokens_seen": 845217792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757071213640923, + "loss": 2.7096, + "theoretical_loss": 3.708957379234331, + "tokens_seen": 845283328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037569709127382143, + "loss": 2.7556, + "theoretical_loss": 3.7089291578594548, + "tokens_seen": 845348864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 474273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4147276878356934, + "objective/train/theoretical_loss": 3.7089009392849173, + "objective/train/tokens_used": 865874400, + "theoretical_loss": 3.7089009392849173, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037568706118355067, + "loss": 2.8156, + "theoretical_loss": 3.7089009392849173, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037567703109327985, + "loss": 2.8181, + "theoretical_loss": 3.7088727235102237, + "tokens_seen": 845479936 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037566700100300903, + "loss": 2.7168, + "theoretical_loss": 3.7088445105348793, + "tokens_seen": 845545472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756569709127382, + "loss": 2.7358, + "theoretical_loss": 3.7088163003583885, + "tokens_seen": 845611008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037564694082246745, + "loss": 2.6648, + "theoretical_loss": 3.7087880929802575, + "tokens_seen": 845676544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756369107321966, + "loss": 2.7797, + "theoretical_loss": 3.708759888399992, + "tokens_seen": 845742080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756268806419258, + "loss": 2.7489, + "theoretical_loss": 3.708731686617097, + "tokens_seen": 845807616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037561685055165494, + "loss": 2.6411, + "theoretical_loss": 3.7087034876310794, + "tokens_seen": 845873152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037560682046138417, + "loss": 2.5334, + "theoretical_loss": 3.7086752914414443, + "tokens_seen": 845938688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037559679037111335, + "loss": 2.5147, + "theoretical_loss": 3.708647098047698, + "tokens_seen": 846004224 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037558676028084253, + "loss": 2.771, + "theoretical_loss": 3.708618907449347, + "tokens_seen": 846069760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755767301905717, + "loss": 2.8255, + "theoretical_loss": 3.7085907196458976, + "tokens_seen": 846135296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755667001003009, + "loss": 2.8221, + "theoretical_loss": 3.7085625346368554, + "tokens_seen": 846200832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755566700100301, + "loss": 2.6886, + "theoretical_loss": 3.7085343524217285, + "tokens_seen": 846266368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755466399197593, + "loss": 2.5909, + "theoretical_loss": 3.708506173000022, + "tokens_seen": 846331904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037553660982948844, + "loss": 2.6621, + "theoretical_loss": 3.7084779963712435, + "tokens_seen": 846397440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755265797392177, + "loss": 2.4955, + "theoretical_loss": 3.7084498225349005, + "tokens_seen": 846462976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755165496489468, + "loss": 2.7909, + "theoretical_loss": 3.7084216514904993, + "tokens_seen": 846528512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037550651955867604, + "loss": 2.7867, + "theoretical_loss": 3.7083934832375474, + "tokens_seen": 846594048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754964894684052, + "loss": 2.63, + "theoretical_loss": 3.7083653177755522, + "tokens_seen": 846659584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754864593781344, + "loss": 2.647, + "theoretical_loss": 3.708337155104021, + "tokens_seen": 846725120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754764292878636, + "loss": 2.5783, + "theoretical_loss": 3.7083089952224615, + "tokens_seen": 846790656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754663991975928, + "loss": 2.7086, + "theoretical_loss": 3.7082808381303813, + "tokens_seen": 846856192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037545636910732194, + "loss": 2.8197, + "theoretical_loss": 3.7082526838272885, + "tokens_seen": 846921728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754463390170512, + "loss": 2.6207, + "theoretical_loss": 3.7082245323126903, + "tokens_seen": 846987264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 475572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3531529903411865, + "objective/train/theoretical_loss": 3.7081963835860963, + "objective/train/tokens_used": 867512800, + "theoretical_loss": 3.7081963835860963, + "tokens_seen": 847052800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754363089267803, + "loss": 2.3995, + "theoretical_loss": 3.7081963835860963, + "tokens_seen": 847052800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037542627883650954, + "loss": 2.7983, + "theoretical_loss": 3.7081682376470133, + "tokens_seen": 847118336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754162487462387, + "loss": 2.8877, + "theoretical_loss": 3.7081400944949503, + "tokens_seen": 847183872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754062186559679, + "loss": 2.7212, + "theoretical_loss": 3.708111954129415, + "tokens_seen": 847249408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753961885656971, + "loss": 2.747, + "theoretical_loss": 3.7080838165499177, + "tokens_seen": 847314944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037538615847542626, + "loss": 2.8743, + "theoretical_loss": 3.7080556817559653, + "tokens_seen": 847380480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037537612838515544, + "loss": 2.7958, + "theoretical_loss": 3.7080275497470674, + "tokens_seen": 847446016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753660982948847, + "loss": 2.5247, + "theoretical_loss": 3.707999420522733, + "tokens_seen": 847511552 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037535606820461386, + "loss": 2.8099, + "theoretical_loss": 3.707971294082471, + "tokens_seen": 847577088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037534603811434304, + "loss": 2.8774, + "theoretical_loss": 3.7079431704257906, + "tokens_seen": 847642624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753360080240723, + "loss": 2.8221, + "theoretical_loss": 3.7079150495522013, + "tokens_seen": 847708160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753259779338014, + "loss": 2.7504, + "theoretical_loss": 3.7078869314612124, + "tokens_seen": 847773696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037531594784353064, + "loss": 2.5578, + "theoretical_loss": 3.707858816152334, + "tokens_seen": 847839232 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037530591775325977, + "loss": 2.8172, + "theoretical_loss": 3.707830703625075, + "tokens_seen": 847904768 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375295887662989, + "loss": 2.7628, + "theoretical_loss": 3.7078025938789456, + "tokens_seen": 847970304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752858575727182, + "loss": 2.4287, + "theoretical_loss": 3.707774486913456, + "tokens_seen": 848035840 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037527582748244736, + "loss": 2.8295, + "theoretical_loss": 3.707746382728116, + "tokens_seen": 848101376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037526579739217655, + "loss": 2.6956, + "theoretical_loss": 3.7077182813224354, + "tokens_seen": 848166912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752557673019057, + "loss": 2.5746, + "theoretical_loss": 3.7076901826959254, + "tokens_seen": 848232448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752457372116349, + "loss": 2.4742, + "theoretical_loss": 3.7076620868480963, + "tokens_seen": 848297984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037523570712136414, + "loss": 2.6048, + "theoretical_loss": 3.707633993778458, + "tokens_seen": 848363520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037522567703109327, + "loss": 2.8559, + "theoretical_loss": 3.707605903486522, + "tokens_seen": 848429056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752156469408225, + "loss": 2.4246, + "theoretical_loss": 3.7075778159717983, + "tokens_seen": 848494592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037520561685055163, + "loss": 2.7434, + "theoretical_loss": 3.7075497312337986, + "tokens_seen": 848560128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037519558676028087, + "loss": 2.717, + "theoretical_loss": 3.7075216492720338, + "tokens_seen": 848625664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 476007, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1800570487976074, + "objective/train/theoretical_loss": 3.7074935700860143, + "objective/train/tokens_used": 869151200, + "theoretical_loss": 3.7074935700860143, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037518555667001005, + "loss": 2.9501, + "theoretical_loss": 3.7074935700860143, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037517552657973923, + "loss": 2.5482, + "theoretical_loss": 3.7074654936752527, + "tokens_seen": 848756736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751654964894684, + "loss": 2.6684, + "theoretical_loss": 3.70743742003926, + "tokens_seen": 848822272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037515546639919765, + "loss": 2.66, + "theoretical_loss": 3.7074093491775475, + "tokens_seen": 848887808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751454363089268, + "loss": 2.5614, + "theoretical_loss": 3.7073812810896265, + "tokens_seen": 848953344 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375135406218656, + "loss": 2.7405, + "theoretical_loss": 3.7073532157750098, + "tokens_seen": 849018880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037512537612838514, + "loss": 2.7963, + "theoretical_loss": 3.7073251532332088, + "tokens_seen": 849084416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037511534603811437, + "loss": 2.7726, + "theoretical_loss": 3.707297093463735, + "tokens_seen": 849149952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037510531594784355, + "loss": 2.526, + "theoretical_loss": 3.7072690364661023, + "tokens_seen": 849215488 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037509528585757273, + "loss": 2.7597, + "theoretical_loss": 3.707240982239821, + "tokens_seen": 849281024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750852557673019, + "loss": 2.6971, + "theoretical_loss": 3.7072129307844053, + "tokens_seen": 849346560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750752256770311, + "loss": 2.8605, + "theoretical_loss": 3.707184882099366, + "tokens_seen": 849412096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750651955867603, + "loss": 2.5104, + "theoretical_loss": 3.7071568361842173, + "tokens_seen": 849477632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750551654964895, + "loss": 2.5936, + "theoretical_loss": 3.7071287930384713, + "tokens_seen": 849543168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037504513540621864, + "loss": 2.5493, + "theoretical_loss": 3.7071007526616406, + "tokens_seen": 849608704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750351053159479, + "loss": 2.6672, + "theoretical_loss": 3.707072715053239, + "tokens_seen": 849674240 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375025075225677, + "loss": 2.8005, + "theoretical_loss": 3.707044680212779, + "tokens_seen": 849739776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037501504513540624, + "loss": 2.7663, + "theoretical_loss": 3.707016648139775, + "tokens_seen": 849805312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750050150451354, + "loss": 2.8586, + "theoretical_loss": 3.7069886188337393, + "tokens_seen": 849870848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749949849548646, + "loss": 2.801, + "theoretical_loss": 3.7069605922941857, + "tokens_seen": 849936384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749849548645938, + "loss": 2.865, + "theoretical_loss": 3.706932568520628, + "tokens_seen": 850001920 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374974924774323, + "loss": 2.7011, + "theoretical_loss": 3.70690454751258, + "tokens_seen": 850067456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037496489468405214, + "loss": 2.7869, + "theoretical_loss": 3.7068765292695556, + "tokens_seen": 850132992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749548645937814, + "loss": 2.889, + "theoretical_loss": 3.7068485137910683, + "tokens_seen": 850198528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749448345035105, + "loss": 2.6505, + "theoretical_loss": 3.7068205010766335, + "tokens_seen": 850264064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 477422, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5388436317443848, + "objective/train/theoretical_loss": 3.7067924911257646, + "objective/train/tokens_used": 870789600, + "theoretical_loss": 3.7067924911257646, + "tokens_seen": 850329600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037493480441323974, + "loss": 2.5794, + "theoretical_loss": 3.7067924911257646, + "tokens_seen": 850329600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749247743229689, + "loss": 2.5744, + "theoretical_loss": 3.7067644839379756, + "tokens_seen": 850395136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749147442326981, + "loss": 2.7938, + "theoretical_loss": 3.706736479512782, + "tokens_seen": 850460672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749047141424273, + "loss": 2.7906, + "theoretical_loss": 3.7067084778496984, + "tokens_seen": 850526208 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037489468405215646, + "loss": 2.7479, + "theoretical_loss": 3.7066804789482384, + "tokens_seen": 850591744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037488465396188564, + "loss": 2.5192, + "theoretical_loss": 3.706652482807918, + "tokens_seen": 850657280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748746238716149, + "loss": 2.5515, + "theoretical_loss": 3.7066244894282523, + "tokens_seen": 850722816 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374864593781344, + "loss": 2.8724, + "theoretical_loss": 3.7065964988087554, + "tokens_seen": 850788352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037485456369107324, + "loss": 2.6995, + "theoretical_loss": 3.7065685109489435, + "tokens_seen": 850853888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037484453360080237, + "loss": 2.5975, + "theoretical_loss": 3.7065405258483315, + "tokens_seen": 850919424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748345035105316, + "loss": 2.9188, + "theoretical_loss": 3.7065125435064354, + "tokens_seen": 850984960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748244734202608, + "loss": 2.7731, + "theoretical_loss": 3.7064845639227704, + "tokens_seen": 851050496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037481444332998997, + "loss": 2.7823, + "theoretical_loss": 3.706456587096852, + "tokens_seen": 851116032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037480441323971915, + "loss": 2.5889, + "theoretical_loss": 3.706428613028197, + "tokens_seen": 851181568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747943831494484, + "loss": 2.7635, + "theoretical_loss": 3.706400641716321, + "tokens_seen": 851247104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747843530591775, + "loss": 2.6144, + "theoretical_loss": 3.7063726731607396, + "tokens_seen": 851312640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037477432296890675, + "loss": 2.663, + "theoretical_loss": 3.7063447073609694, + "tokens_seen": 851378176 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037476429287863587, + "loss": 2.6883, + "theoretical_loss": 3.706316744316527, + "tokens_seen": 851443712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747542627883651, + "loss": 2.6976, + "theoretical_loss": 3.7062887840269276, + "tokens_seen": 851509248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747442326980943, + "loss": 2.778, + "theoretical_loss": 3.7062608264916896, + "tokens_seen": 851574784 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037473420260782347, + "loss": 2.7051, + "theoretical_loss": 3.7062328717103292, + "tokens_seen": 851640320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037472417251755265, + "loss": 2.8306, + "theoretical_loss": 3.7062049196823628, + "tokens_seen": 851705856 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037471414242728183, + "loss": 2.4378, + "theoretical_loss": 3.7061769704073075, + "tokens_seen": 851771392 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374704112337011, + "loss": 2.5944, + "theoretical_loss": 3.706149023884681, + "tokens_seen": 851836928 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037469408224674025, + "loss": 2.783, + "theoretical_loss": 3.7061210801139994, + "tokens_seen": 851902464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 478724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.294400453567505, + "objective/train/theoretical_loss": 3.706093139094781, + "objective/train/tokens_used": 872428000, + "theoretical_loss": 3.706093139094781, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746840521564694, + "loss": 2.5393, + "theoretical_loss": 3.706093139094781, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746740220661986, + "loss": 2.7337, + "theoretical_loss": 3.706065200826543, + "tokens_seen": 852033536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037466399197592774, + "loss": 2.5102, + "theoretical_loss": 3.7060372653088027, + "tokens_seen": 852099072 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374653961885657, + "loss": 2.5369, + "theoretical_loss": 3.706009332541078, + "tokens_seen": 852164608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037464393179538615, + "loss": 2.7483, + "theoretical_loss": 3.7059814025228865, + "tokens_seen": 852230144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037463390170511534, + "loss": 2.7406, + "theoretical_loss": 3.7059534752537466, + "tokens_seen": 852295680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037462387161484457, + "loss": 2.7608, + "theoretical_loss": 3.705925550733176, + "tokens_seen": 852361216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037461384152457375, + "loss": 2.7804, + "theoretical_loss": 3.7058976289606935, + "tokens_seen": 852426752 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037460381143430293, + "loss": 2.7932, + "theoretical_loss": 3.7058697099358167, + "tokens_seen": 852492288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745937813440321, + "loss": 2.7504, + "theoretical_loss": 3.705841793658064, + "tokens_seen": 852557824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745837512537613, + "loss": 2.8511, + "theoretical_loss": 3.7058138801269545, + "tokens_seen": 852623360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745737211634905, + "loss": 2.6997, + "theoretical_loss": 3.7057859693420063, + "tokens_seen": 852688896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745636910732197, + "loss": 2.8532, + "theoretical_loss": 3.7057580613027383, + "tokens_seen": 852754432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037455366098294884, + "loss": 2.5094, + "theoretical_loss": 3.70573015600867, + "tokens_seen": 852819968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745436308926781, + "loss": 2.8309, + "theoretical_loss": 3.70570225345932, + "tokens_seen": 852885504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745336008024072, + "loss": 2.7093, + "theoretical_loss": 3.7056743536542074, + "tokens_seen": 852951040 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037452357071213644, + "loss": 2.8218, + "theoretical_loss": 3.7056464565928517, + "tokens_seen": 853016576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745135406218656, + "loss": 2.5999, + "theoretical_loss": 3.705618562274772, + "tokens_seen": 853082112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745035105315948, + "loss": 2.562, + "theoretical_loss": 3.7055906706994883, + "tokens_seen": 853147648 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374493480441324, + "loss": 2.7136, + "theoretical_loss": 3.7055627818665196, + "tokens_seen": 853213184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744834503510532, + "loss": 2.7561, + "theoretical_loss": 3.7055348957753864, + "tokens_seen": 853278720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037447342026078234, + "loss": 2.7161, + "theoretical_loss": 3.705507012425608, + "tokens_seen": 853344256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744633901705116, + "loss": 2.7196, + "theoretical_loss": 3.7054791318167046, + "tokens_seen": 853409792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744533600802407, + "loss": 2.8581, + "theoretical_loss": 3.7054512539481967, + "tokens_seen": 853475328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037444332998996994, + "loss": 2.7128, + "theoretical_loss": 3.705423378819604, + "tokens_seen": 853540864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 479323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.415107488632202, + "objective/train/theoretical_loss": 3.7053955064304467, + "objective/train/tokens_used": 874066400, + "theoretical_loss": 3.7053955064304467, + "tokens_seen": 853606400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744332998996991, + "loss": 2.7135, + "theoretical_loss": 3.7053955064304467, + "tokens_seen": 853606400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744232698094283, + "loss": 2.5923, + "theoretical_loss": 3.705367636780246, + "tokens_seen": 853671936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744132397191575, + "loss": 2.6997, + "theoretical_loss": 3.705339769868522, + "tokens_seen": 853737472 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037440320962888666, + "loss": 2.777, + "theoretical_loss": 3.7053119056947956, + "tokens_seen": 853803008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037439317953861585, + "loss": 2.5342, + "theoretical_loss": 3.7052840442585877, + "tokens_seen": 853868544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743831494483451, + "loss": 2.7932, + "theoretical_loss": 3.7052561855594193, + "tokens_seen": 853934080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743731193580742, + "loss": 2.5486, + "theoretical_loss": 3.705228329596811, + "tokens_seen": 853999616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037436308926780344, + "loss": 2.8331, + "theoretical_loss": 3.705200476370285, + "tokens_seen": 854065152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037435305917753257, + "loss": 2.6312, + "theoretical_loss": 3.7051726258793622, + "tokens_seen": 854130688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743430290872618, + "loss": 2.6545, + "theoretical_loss": 3.7051447781235636, + "tokens_seen": 854196224 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374332998996991, + "loss": 2.6804, + "theoretical_loss": 3.7051169331024107, + "tokens_seen": 854261760 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037432296890672017, + "loss": 2.646, + "theoretical_loss": 3.7050890908154264, + "tokens_seen": 854327296 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037431293881644935, + "loss": 2.5474, + "theoretical_loss": 3.7050612512621313, + "tokens_seen": 854392832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743029087261786, + "loss": 2.6389, + "theoretical_loss": 3.7050334144420476, + "tokens_seen": 854458368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742928786359077, + "loss": 2.6941, + "theoretical_loss": 3.7050055803546975, + "tokens_seen": 854523904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037428284854563695, + "loss": 2.6424, + "theoretical_loss": 3.7049777489996036, + "tokens_seen": 854589440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037427281845536607, + "loss": 2.6623, + "theoretical_loss": 3.7049499203762872, + "tokens_seen": 854654976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742627883650953, + "loss": 2.6297, + "theoretical_loss": 3.704922094484272, + "tokens_seen": 854720512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742527582748245, + "loss": 2.774, + "theoretical_loss": 3.7048942713230795, + "tokens_seen": 854786048 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037424272818455367, + "loss": 2.73, + "theoretical_loss": 3.7048664508922324, + "tokens_seen": 854851584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037423269809428285, + "loss": 2.7748, + "theoretical_loss": 3.704838633191254, + "tokens_seen": 854917120 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037422266800401203, + "loss": 2.6596, + "theoretical_loss": 3.7048108182196673, + "tokens_seen": 854982656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742126379137412, + "loss": 2.7619, + "theoretical_loss": 3.704783005976995, + "tokens_seen": 855048192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037420260782347045, + "loss": 2.7137, + "theoretical_loss": 3.7047551964627594, + "tokens_seen": 855113728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741925777331996, + "loss": 2.8362, + "theoretical_loss": 3.704727389676486, + "tokens_seen": 855179264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 479893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.8994100093841553, + "objective/train/theoretical_loss": 3.7046995856176954, + "objective/train/tokens_used": 875704800, + "theoretical_loss": 3.7046995856176954, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741825476429288, + "loss": 2.3327, + "theoretical_loss": 3.7046995856176954, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037417251755265794, + "loss": 2.7426, + "theoretical_loss": 3.704671784285913, + "tokens_seen": 855310336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741624874623872, + "loss": 2.7, + "theoretical_loss": 3.704643985680662, + "tokens_seen": 855375872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037415245737211635, + "loss": 2.7171, + "theoretical_loss": 3.704616189801466, + "tokens_seen": 855441408 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037414242728184554, + "loss": 2.7728, + "theoretical_loss": 3.704588396647848, + "tokens_seen": 855506944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741323971915747, + "loss": 2.7179, + "theoretical_loss": 3.704560606219334, + "tokens_seen": 855572480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037412236710130395, + "loss": 2.4159, + "theoretical_loss": 3.7045328185154465, + "tokens_seen": 855638016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741123370110331, + "loss": 2.6816, + "theoretical_loss": 3.7045050335357104, + "tokens_seen": 855703552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741023069207623, + "loss": 2.7457, + "theoretical_loss": 3.7044772512796493, + "tokens_seen": 855769088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037409227683049144, + "loss": 2.4778, + "theoretical_loss": 3.7044494717467886, + "tokens_seen": 855834624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740822467402207, + "loss": 2.759, + "theoretical_loss": 3.704421694936652, + "tokens_seen": 855900160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037407221664994986, + "loss": 2.7427, + "theoretical_loss": 3.704393920848765, + "tokens_seen": 855965696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037406218655967904, + "loss": 2.8548, + "theoretical_loss": 3.704366149482652, + "tokens_seen": 856031232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740521564694082, + "loss": 2.6829, + "theoretical_loss": 3.7043383808378376, + "tokens_seen": 856096768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740421263791374, + "loss": 2.4935, + "theoretical_loss": 3.7043106149138474, + "tokens_seen": 856162304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740320962888666, + "loss": 2.4234, + "theoretical_loss": 3.7042828517102064, + "tokens_seen": 856227840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740220661985958, + "loss": 2.6602, + "theoretical_loss": 3.7042550912264405, + "tokens_seen": 856293376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037401203610832494, + "loss": 2.6378, + "theoretical_loss": 3.7042273334620734, + "tokens_seen": 856358912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740020060180542, + "loss": 2.5173, + "theoretical_loss": 3.7041995784166324, + "tokens_seen": 856424448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037399197592778336, + "loss": 2.8294, + "theoretical_loss": 3.7041718260896426, + "tokens_seen": 856489984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037398194583751254, + "loss": 2.7361, + "theoretical_loss": 3.7041440764806293, + "tokens_seen": 856555520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739719157472417, + "loss": 2.9064, + "theoretical_loss": 3.7041163295891186, + "tokens_seen": 856621056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739618856569709, + "loss": 2.7333, + "theoretical_loss": 3.7040885854146364, + "tokens_seen": 856686592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739518555667001, + "loss": 2.7232, + "theoretical_loss": 3.7040608439567095, + "tokens_seen": 856752128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739418254764293, + "loss": 2.6206, + "theoretical_loss": 3.7040331052148634, + "tokens_seen": 856817664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 480416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0554239749908447, + "objective/train/theoretical_loss": 3.704005369188625, + "objective/train/tokens_used": 877343200, + "theoretical_loss": 3.704005369188625, + "tokens_seen": 856883200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037393179538615845, + "loss": 2.8683, + "theoretical_loss": 3.704005369188625, + "tokens_seen": 856883200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739217652958877, + "loss": 2.8753, + "theoretical_loss": 3.70397763587752, + "tokens_seen": 856948736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739117352056168, + "loss": 2.4591, + "theoretical_loss": 3.7039499052810765, + "tokens_seen": 857014272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037390170511534605, + "loss": 2.7519, + "theoretical_loss": 3.703922177398819, + "tokens_seen": 857079808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738916750250752, + "loss": 2.7931, + "theoretical_loss": 3.7038944522302764, + "tokens_seen": 857145344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738816449348044, + "loss": 2.6563, + "theoretical_loss": 3.7038667297749743, + "tokens_seen": 857210880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037387161484453364, + "loss": 2.6392, + "theoretical_loss": 3.703839010032441, + "tokens_seen": 857276416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037386158475426277, + "loss": 2.6964, + "theoretical_loss": 3.703811293002202, + "tokens_seen": 857341952 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373851554663992, + "loss": 2.5881, + "theoretical_loss": 3.7037835786837867, + "tokens_seen": 857407488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738415245737212, + "loss": 2.697, + "theoretical_loss": 3.7037558670767208, + "tokens_seen": 857473024 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037383149448345037, + "loss": 2.611, + "theoretical_loss": 3.7037281581805326, + "tokens_seen": 857538560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037382146439317955, + "loss": 2.5995, + "theoretical_loss": 3.7037004519947496, + "tokens_seen": 857604096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738114343029088, + "loss": 2.3132, + "theoretical_loss": 3.7036727485188994, + "tokens_seen": 857669632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738014042126379, + "loss": 2.7532, + "theoretical_loss": 3.7036450477525102, + "tokens_seen": 857735168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037379137412236715, + "loss": 2.7108, + "theoretical_loss": 3.7036173496951106, + "tokens_seen": 857800704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037378134403209627, + "loss": 2.9879, + "theoretical_loss": 3.703589654346227, + "tokens_seen": 857866240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737713139418255, + "loss": 2.9101, + "theoretical_loss": 3.7035619617053896, + "tokens_seen": 857931776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737612838515547, + "loss": 2.6775, + "theoretical_loss": 3.703534271772125, + "tokens_seen": 857997312 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037375125376128387, + "loss": 2.6034, + "theoretical_loss": 3.7035065845459636, + "tokens_seen": 858062848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037374122367101305, + "loss": 2.6527, + "theoretical_loss": 3.7034789000264325, + "tokens_seen": 858128384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037373119358074223, + "loss": 2.7135, + "theoretical_loss": 3.7034512182130612, + "tokens_seen": 858193920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737211634904714, + "loss": 2.5494, + "theoretical_loss": 3.703423539105378, + "tokens_seen": 858259456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037371113340020065, + "loss": 2.7046, + "theoretical_loss": 3.703395862702912, + "tokens_seen": 858324992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737011033099298, + "loss": 2.5938, + "theoretical_loss": 3.703368189005193, + "tokens_seen": 858390528 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373691073219659, + "loss": 2.5671, + "theoretical_loss": 3.70334051801175, + "tokens_seen": 858456064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 481568, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5986602306365967, + "objective/train/theoretical_loss": 3.703312849722111, + "objective/train/tokens_used": 878981600, + "theoretical_loss": 3.703312849722111, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037368104312938814, + "loss": 2.8, + "theoretical_loss": 3.703312849722111, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736710130391174, + "loss": 2.6659, + "theoretical_loss": 3.7032851841358063, + "tokens_seen": 858587136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037366098294884655, + "loss": 2.431, + "theoretical_loss": 3.7032575212523664, + "tokens_seen": 858652672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037365095285857574, + "loss": 2.788, + "theoretical_loss": 3.7032298610713195, + "tokens_seen": 858718208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736409227683049, + "loss": 2.5209, + "theoretical_loss": 3.7032022035921965, + "tokens_seen": 858783744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037363089267803415, + "loss": 2.6643, + "theoretical_loss": 3.7031745488145265, + "tokens_seen": 858849280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736208625877633, + "loss": 2.8994, + "theoretical_loss": 3.70314689673784, + "tokens_seen": 858914816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736108324974925, + "loss": 2.9014, + "theoretical_loss": 3.703119247361667, + "tokens_seen": 858980352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037360080240722164, + "loss": 2.8771, + "theoretical_loss": 3.7030916006855374, + "tokens_seen": 859045888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735907723169509, + "loss": 2.8042, + "theoretical_loss": 3.7030639567089825, + "tokens_seen": 859111424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037358074222668006, + "loss": 2.6583, + "theoretical_loss": 3.7030363154315316, + "tokens_seen": 859176960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037357071213640924, + "loss": 2.6478, + "theoretical_loss": 3.7030086768527166, + "tokens_seen": 859242496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735606820461384, + "loss": 2.5504, + "theoretical_loss": 3.702981040972067, + "tokens_seen": 859308032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735506519558676, + "loss": 2.7879, + "theoretical_loss": 3.7029534077891144, + "tokens_seen": 859373568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735406218655968, + "loss": 2.9783, + "theoretical_loss": 3.70292577730339, + "tokens_seen": 859439104 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373530591775326, + "loss": 2.9209, + "theoretical_loss": 3.702898149514424, + "tokens_seen": 859504640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037352056168505514, + "loss": 2.592, + "theoretical_loss": 3.702870524421748, + "tokens_seen": 859570176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735105315947844, + "loss": 2.816, + "theoretical_loss": 3.7028429020248934, + "tokens_seen": 859635712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037350050150451356, + "loss": 2.4548, + "theoretical_loss": 3.7028152823233924, + "tokens_seen": 859701248 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037349047141424274, + "loss": 2.8427, + "theoretical_loss": 3.702787665316775, + "tokens_seen": 859766784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734804413239719, + "loss": 2.6741, + "theoretical_loss": 3.7027600510045735, + "tokens_seen": 859832320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734704112337011, + "loss": 2.844, + "theoretical_loss": 3.7027324393863204, + "tokens_seen": 859897856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734603811434303, + "loss": 2.8788, + "theoretical_loss": 3.702704830461547, + "tokens_seen": 859963392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734503510531595, + "loss": 2.5995, + "theoretical_loss": 3.702677224229785, + "tokens_seen": 860028928 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037344032096288865, + "loss": 2.7775, + "theoretical_loss": 3.702649620690567, + "tokens_seen": 860094464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 482749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.050727367401123, + "objective/train/theoretical_loss": 3.7026220198434254, + "objective/train/tokens_used": 880620000, + "theoretical_loss": 3.7026220198434254, + "tokens_seen": 860160000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734302908726179, + "loss": 2.7565, + "theoretical_loss": 3.7026220198434254, + "tokens_seen": 860160000 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373420260782347, + "loss": 2.5725, + "theoretical_loss": 3.702594421687892, + "tokens_seen": 860225536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037341023069207625, + "loss": 2.5902, + "theoretical_loss": 3.7025668262234994, + "tokens_seen": 860291072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734002006018054, + "loss": 2.8756, + "theoretical_loss": 3.702539233449781, + "tokens_seen": 860356608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733901705115346, + "loss": 2.4553, + "theoretical_loss": 3.702511643366268, + "tokens_seen": 860422144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733801404212638, + "loss": 2.7468, + "theoretical_loss": 3.702484055972495, + "tokens_seen": 860487680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037337011033099297, + "loss": 2.7126, + "theoretical_loss": 3.702456471267994, + "tokens_seen": 860553216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037336008024072215, + "loss": 2.7523, + "theoretical_loss": 3.7024288892522983, + "tokens_seen": 860618752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733500501504514, + "loss": 2.6144, + "theoretical_loss": 3.702401309924941, + "tokens_seen": 860684288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733400200601805, + "loss": 2.5013, + "theoretical_loss": 3.702373733285455, + "tokens_seen": 860749824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037332998996990975, + "loss": 2.5324, + "theoretical_loss": 3.7023461593333744, + "tokens_seen": 860815360 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037331995987963893, + "loss": 2.6501, + "theoretical_loss": 3.7023185880682323, + "tokens_seen": 860880896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733099297893681, + "loss": 2.8881, + "theoretical_loss": 3.702291019489562, + "tokens_seen": 860946432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732998996990973, + "loss": 2.7107, + "theoretical_loss": 3.7022634535968986, + "tokens_seen": 861011968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732898696088265, + "loss": 2.8248, + "theoretical_loss": 3.7022358903897747, + "tokens_seen": 861077504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037327983951855565, + "loss": 2.6673, + "theoretical_loss": 3.702208329867725, + "tokens_seen": 861143040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732698094282849, + "loss": 2.7935, + "theoretical_loss": 3.7021807720302835, + "tokens_seen": 861208576 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373259779338014, + "loss": 2.6674, + "theoretical_loss": 3.702153216876984, + "tokens_seen": 861274112 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037324974924774325, + "loss": 2.7553, + "theoretical_loss": 3.702125664407361, + "tokens_seen": 861339648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732397191574724, + "loss": 2.7629, + "theoretical_loss": 3.7020981146209495, + "tokens_seen": 861405184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732296890672016, + "loss": 2.6363, + "theoretical_loss": 3.7020705675172834, + "tokens_seen": 861470720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732196589769308, + "loss": 2.9832, + "theoretical_loss": 3.702043023095898, + "tokens_seen": 861536256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037320962888666, + "loss": 2.7581, + "theoretical_loss": 3.7020154813563275, + "tokens_seen": 861601792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037319959879638916, + "loss": 2.6392, + "theoretical_loss": 3.7019879422981075, + "tokens_seen": 861667328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037318956870611834, + "loss": 2.7417, + "theoretical_loss": 3.7019604059207727, + "tokens_seen": 861732864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 483414, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.466357707977295, + "objective/train/theoretical_loss": 3.701932872223858, + "objective/train/tokens_used": 882258400, + "theoretical_loss": 3.701932872223858, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731795386158475, + "loss": 2.7092, + "theoretical_loss": 3.701932872223858, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037316950852557675, + "loss": 2.5774, + "theoretical_loss": 3.701905341206899, + "tokens_seen": 861863936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731594784353059, + "loss": 2.5797, + "theoretical_loss": 3.701877812869431, + "tokens_seen": 861929472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731494483450351, + "loss": 2.4721, + "theoretical_loss": 3.70185028721099, + "tokens_seen": 861995008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731394182547643, + "loss": 2.6781, + "theoretical_loss": 3.7018227642311112, + "tokens_seen": 862060544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731293881644935, + "loss": 2.6883, + "theoretical_loss": 3.70179524392933, + "tokens_seen": 862126080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731193580742227, + "loss": 2.7031, + "theoretical_loss": 3.7017677263051825, + "tokens_seen": 862191616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037310932798395184, + "loss": 2.7407, + "theoretical_loss": 3.701740211358205, + "tokens_seen": 862257152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730992978936811, + "loss": 2.8633, + "theoretical_loss": 3.7017126990879334, + "tokens_seen": 862322688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037308926780341026, + "loss": 2.856, + "theoretical_loss": 3.7016851894939036, + "tokens_seen": 862388224 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037307923771313944, + "loss": 2.7041, + "theoretical_loss": 3.701657682575653, + "tokens_seen": 862453760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730692076228686, + "loss": 2.8978, + "theoretical_loss": 3.7016301783327163, + "tokens_seen": 862519296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730591775325978, + "loss": 2.5655, + "theoretical_loss": 3.7016026767646313, + "tokens_seen": 862584832 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373049147442327, + "loss": 2.4688, + "theoretical_loss": 3.7015751778709345, + "tokens_seen": 862650368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730391173520562, + "loss": 2.8267, + "theoretical_loss": 3.701547681651162, + "tokens_seen": 862715904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037302908726178534, + "loss": 2.7697, + "theoretical_loss": 3.701520188104852, + "tokens_seen": 862781440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730190571715146, + "loss": 2.7066, + "theoretical_loss": 3.70149269723154, + "tokens_seen": 862846976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037300902708124376, + "loss": 2.8466, + "theoretical_loss": 3.7014652090307645, + "tokens_seen": 862912512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037299899699097294, + "loss": 2.5227, + "theoretical_loss": 3.701437723502062, + "tokens_seen": 862978048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729889669007021, + "loss": 2.7204, + "theoretical_loss": 3.70141024064497, + "tokens_seen": 863043584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729789368104313, + "loss": 2.5038, + "theoretical_loss": 3.7013827604590253, + "tokens_seen": 863109120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729689067201605, + "loss": 2.7124, + "theoretical_loss": 3.7013552829437675, + "tokens_seen": 863174656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729588766298897, + "loss": 2.8171, + "theoretical_loss": 3.701327808098732, + "tokens_seen": 863240192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037294884653961885, + "loss": 2.9852, + "theoretical_loss": 3.7013003359234578, + "tokens_seen": 863305728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729388164493481, + "loss": 2.6671, + "theoretical_loss": 3.701272866417482, + "tokens_seen": 863371264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 484154, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.915800094604492, + "objective/train/theoretical_loss": 3.701245399580344, + "objective/train/tokens_used": 883896800, + "theoretical_loss": 3.701245399580344, + "tokens_seen": 863436800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729287863590772, + "loss": 2.7206, + "theoretical_loss": 3.701245399580344, + "tokens_seen": 863436800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037291875626880645, + "loss": 2.7457, + "theoretical_loss": 3.7012179354115813, + "tokens_seen": 863502336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729087261785356, + "loss": 2.8516, + "theoretical_loss": 3.701190473910732, + "tokens_seen": 863567872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728986960882648, + "loss": 2.6364, + "theoretical_loss": 3.7011630150773343, + "tokens_seen": 863633408 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372888665997994, + "loss": 2.7432, + "theoretical_loss": 3.7011355589109276, + "tokens_seen": 863698944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037287863590772317, + "loss": 2.6735, + "theoretical_loss": 3.7011081054110493, + "tokens_seen": 863764480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037286860581745235, + "loss": 2.675, + "theoretical_loss": 3.7010806545772397, + "tokens_seen": 863830016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728585757271816, + "loss": 2.8821, + "theoretical_loss": 3.7010532064090356, + "tokens_seen": 863895552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728485456369107, + "loss": 2.5917, + "theoretical_loss": 3.701025760905978, + "tokens_seen": 863961088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037283851554663995, + "loss": 2.761, + "theoretical_loss": 3.7009983180676045, + "tokens_seen": 864026624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037282848545636913, + "loss": 2.8932, + "theoretical_loss": 3.7009708778934556, + "tokens_seen": 864092160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728184553660983, + "loss": 2.8616, + "theoretical_loss": 3.7009434403830697, + "tokens_seen": 864157696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728084252758275, + "loss": 2.7249, + "theoretical_loss": 3.700916005535986, + "tokens_seen": 864223232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727983951855567, + "loss": 2.6967, + "theoretical_loss": 3.7008885733517447, + "tokens_seen": 864288768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037278836509528585, + "loss": 2.6364, + "theoretical_loss": 3.7008611438298855, + "tokens_seen": 864354304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727783350050151, + "loss": 2.8586, + "theoretical_loss": 3.7008337169699477, + "tokens_seen": 864419840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727683049147442, + "loss": 2.7462, + "theoretical_loss": 3.700806292771472, + "tokens_seen": 864485376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037275827482447345, + "loss": 2.6446, + "theoretical_loss": 3.7007788712339966, + "tokens_seen": 864550912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727482447342026, + "loss": 2.58, + "theoretical_loss": 3.700751452357064, + "tokens_seen": 864616448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727382146439318, + "loss": 2.6979, + "theoretical_loss": 3.7007240361402127, + "tokens_seen": 864681984 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372728184553661, + "loss": 2.7701, + "theoretical_loss": 3.700696622582984, + "tokens_seen": 864747520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727181544633902, + "loss": 2.7526, + "theoretical_loss": 3.7006692116849176, + "tokens_seen": 864813056 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037270812437311936, + "loss": 2.7688, + "theoretical_loss": 3.7006418034455546, + "tokens_seen": 864878592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037269809428284854, + "loss": 2.9445, + "theoretical_loss": 3.7006143978644355, + "tokens_seen": 864944128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726880641925777, + "loss": 2.7108, + "theoretical_loss": 3.7005869949411014, + "tokens_seen": 865009664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 484565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.951662063598633, + "objective/train/theoretical_loss": 3.7005595946750924, + "objective/train/tokens_used": 885535200, + "theoretical_loss": 3.7005595946750924, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037267803410230696, + "loss": 2.7365, + "theoretical_loss": 3.7005595946750924, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726680040120361, + "loss": 2.5957, + "theoretical_loss": 3.70053219706595, + "tokens_seen": 865140736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726579739217653, + "loss": 2.8784, + "theoretical_loss": 3.7005048021132163, + "tokens_seen": 865206272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726479438314945, + "loss": 2.6853, + "theoretical_loss": 3.700477409816431, + "tokens_seen": 865271808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726379137412237, + "loss": 3.0309, + "theoretical_loss": 3.7004500201751362, + "tokens_seen": 865337344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037262788365095286, + "loss": 2.9844, + "theoretical_loss": 3.7004226331888734, + "tokens_seen": 865402880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037261785356068204, + "loss": 2.457, + "theoretical_loss": 3.700395248857184, + "tokens_seen": 865468416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726078234704112, + "loss": 2.9597, + "theoretical_loss": 3.7003678671796103, + "tokens_seen": 865533952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037259779338014046, + "loss": 2.7733, + "theoretical_loss": 3.7003404881556934, + "tokens_seen": 865599488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725877632898696, + "loss": 2.6146, + "theoretical_loss": 3.7003131117849755, + "tokens_seen": 865665024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725777331995988, + "loss": 2.8385, + "theoretical_loss": 3.7002857380669987, + "tokens_seen": 865730560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037256770310932795, + "loss": 2.5251, + "theoretical_loss": 3.7002583670013047, + "tokens_seen": 865796096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725576730190572, + "loss": 2.7264, + "theoretical_loss": 3.700230998587437, + "tokens_seen": 865861632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037254764292878636, + "loss": 2.718, + "theoretical_loss": 3.7002036328249366, + "tokens_seen": 865927168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037253761283851554, + "loss": 2.9428, + "theoretical_loss": 3.7001762697133467, + "tokens_seen": 865992704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725275827482447, + "loss": 2.7134, + "theoretical_loss": 3.70014890925221, + "tokens_seen": 866058240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037251755265797396, + "loss": 2.866, + "theoretical_loss": 3.700121551441069, + "tokens_seen": 866123776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725075225677031, + "loss": 2.5316, + "theoretical_loss": 3.7000941962794673, + "tokens_seen": 866189312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724974924774323, + "loss": 2.955, + "theoretical_loss": 3.7000668437669466, + "tokens_seen": 866254848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037248746238716145, + "loss": 2.7666, + "theoretical_loss": 3.7000394939030503, + "tokens_seen": 866320384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724774322968907, + "loss": 2.803, + "theoretical_loss": 3.7000121466873224, + "tokens_seen": 866385920 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037246740220661987, + "loss": 2.553, + "theoretical_loss": 3.6999848021193054, + "tokens_seen": 866451456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037245737211634905, + "loss": 2.7715, + "theoretical_loss": 3.6999574601985428, + "tokens_seen": 866516992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037244734202607823, + "loss": 2.645, + "theoretical_loss": 3.699930120924579, + "tokens_seen": 866582528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724373119358074, + "loss": 2.8014, + "theoretical_loss": 3.699902784296956, + "tokens_seen": 866648064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 485783, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.643681049346924, + "objective/train/theoretical_loss": 3.6998754503152194, + "objective/train/tokens_used": 887173600, + "theoretical_loss": 3.6998754503152194, + "tokens_seen": 866713600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724272818455366, + "loss": 2.8837, + "theoretical_loss": 3.6998754503152194, + "tokens_seen": 866713600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724172517552658, + "loss": 2.6677, + "theoretical_loss": 3.6998481189789114, + "tokens_seen": 866779136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037240722166499495, + "loss": 2.7465, + "theoretical_loss": 3.6998207902875775, + "tokens_seen": 866844672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723971915747242, + "loss": 2.8192, + "theoretical_loss": 3.6997934642407606, + "tokens_seen": 866910208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723871614844533, + "loss": 2.8605, + "theoretical_loss": 3.6997661408380056, + "tokens_seen": 866975744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037237713139418255, + "loss": 2.7537, + "theoretical_loss": 3.6997388200788563, + "tokens_seen": 867041280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723671013039118, + "loss": 2.7422, + "theoretical_loss": 3.6997115019628577, + "tokens_seen": 867106816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723570712136409, + "loss": 2.7609, + "theoretical_loss": 3.699684186489554, + "tokens_seen": 867172352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037234704112337015, + "loss": 2.7326, + "theoretical_loss": 3.6996568736584896, + "tokens_seen": 867237888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037233701103309933, + "loss": 2.6956, + "theoretical_loss": 3.69962956346921, + "tokens_seen": 867303424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723269809428285, + "loss": 2.6659, + "theoretical_loss": 3.69960225592126, + "tokens_seen": 867368960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723169508525577, + "loss": 2.8015, + "theoretical_loss": 3.699574951014183, + "tokens_seen": 867434496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723069207622869, + "loss": 2.6047, + "theoretical_loss": 3.6995476487475267, + "tokens_seen": 867500032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037229689067201605, + "loss": 2.6313, + "theoretical_loss": 3.6995203491208346, + "tokens_seen": 867565568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722868605817453, + "loss": 2.7724, + "theoretical_loss": 3.6994930521336524, + "tokens_seen": 867631104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722768304914744, + "loss": 2.7818, + "theoretical_loss": 3.699465757785526, + "tokens_seen": 867696640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037226680040120365, + "loss": 2.6688, + "theoretical_loss": 3.699438466076, + "tokens_seen": 867762176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722567703109328, + "loss": 2.6619, + "theoretical_loss": 3.699411177004621, + "tokens_seen": 867827712 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372246740220662, + "loss": 2.7528, + "theoretical_loss": 3.6993838905709344, + "tokens_seen": 867893248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722367101303912, + "loss": 2.6117, + "theoretical_loss": 3.6993566067744856, + "tokens_seen": 867958784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722266800401204, + "loss": 2.5744, + "theoretical_loss": 3.699329325614822, + "tokens_seen": 868024320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037221664994984956, + "loss": 2.6486, + "theoretical_loss": 3.6993020470914884, + "tokens_seen": 868089856 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037220661985957874, + "loss": 2.8173, + "theoretical_loss": 3.6992747712040317, + "tokens_seen": 868155392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721965897693079, + "loss": 2.6862, + "theoretical_loss": 3.699247497951998, + "tokens_seen": 868220928 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037218655967903716, + "loss": 2.7351, + "theoretical_loss": 3.699220227334934, + "tokens_seen": 868286464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 486350, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.766869068145752, + "objective/train/theoretical_loss": 3.699192959352386, + "objective/train/tokens_used": 888812000, + "theoretical_loss": 3.699192959352386, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721765295887663, + "loss": 2.5503, + "theoretical_loss": 3.699192959352386, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721664994984955, + "loss": 3.0311, + "theoretical_loss": 3.6991656940039004, + "tokens_seen": 868417536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721564694082247, + "loss": 2.6552, + "theoretical_loss": 3.6991384312890245, + "tokens_seen": 868483072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721464393179539, + "loss": 2.4007, + "theoretical_loss": 3.6991111712073055, + "tokens_seen": 868548608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037213640922768306, + "loss": 2.809, + "theoretical_loss": 3.69908391375829, + "tokens_seen": 868614144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037212637913741224, + "loss": 2.7626, + "theoretical_loss": 3.699056658941525, + "tokens_seen": 868679680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721163490471414, + "loss": 2.519, + "theoretical_loss": 3.6990294067565577, + "tokens_seen": 868745216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037210631895687066, + "loss": 2.9509, + "theoretical_loss": 3.699002157202936, + "tokens_seen": 868810752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720962888665998, + "loss": 2.7953, + "theoretical_loss": 3.6989749102802065, + "tokens_seen": 868876288 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372086258776329, + "loss": 2.6473, + "theoretical_loss": 3.6989476659879177, + "tokens_seen": 868941824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037207622868605815, + "loss": 2.6791, + "theoretical_loss": 3.6989204243256166, + "tokens_seen": 869007360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720661985957874, + "loss": 2.9017, + "theoretical_loss": 3.698893185292852, + "tokens_seen": 869072896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037205616850551656, + "loss": 2.7375, + "theoretical_loss": 3.6988659488891704, + "tokens_seen": 869138432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037204613841524575, + "loss": 3.008, + "theoretical_loss": 3.698838715114121, + "tokens_seen": 869203968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720361083249749, + "loss": 2.8997, + "theoretical_loss": 3.6988114839672512, + "tokens_seen": 869269504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037202607823470416, + "loss": 2.8079, + "theoretical_loss": 3.6987842554481096, + "tokens_seen": 869335040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720160481444333, + "loss": 2.7345, + "theoretical_loss": 3.6987570295562446, + "tokens_seen": 869400576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720060180541625, + "loss": 2.7938, + "theoretical_loss": 3.6987298062912046, + "tokens_seen": 869466112 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037199598796389165, + "loss": 2.7915, + "theoretical_loss": 3.6987025856525384, + "tokens_seen": 869531648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719859578736209, + "loss": 2.8572, + "theoretical_loss": 3.6986753676397948, + "tokens_seen": 869597184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037197592778335007, + "loss": 2.687, + "theoretical_loss": 3.698648152252521, + "tokens_seen": 869662720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037196589769307925, + "loss": 2.7066, + "theoretical_loss": 3.6986209394902687, + "tokens_seen": 869728256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037195586760280843, + "loss": 2.6519, + "theoretical_loss": 3.6985937293525843, + "tokens_seen": 869793792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719458375125376, + "loss": 2.6958, + "theoretical_loss": 3.6985665218390187, + "tokens_seen": 869859328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719358074222668, + "loss": 2.6778, + "theoretical_loss": 3.6985393169491205, + "tokens_seen": 869924864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 487478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4885756969451904, + "objective/train/theoretical_loss": 3.698512114682439, + "objective/train/tokens_used": 890450400, + "theoretical_loss": 3.698512114682439, + "tokens_seen": 869990400 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371925777331996, + "loss": 2.6768, + "theoretical_loss": 3.698512114682439, + "tokens_seen": 869990400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037191574724172515, + "loss": 2.6099, + "theoretical_loss": 3.698484915038524, + "tokens_seen": 870055936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719057171514544, + "loss": 2.6162, + "theoretical_loss": 3.698457718016925, + "tokens_seen": 870121472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718956870611835, + "loss": 3.0378, + "theoretical_loss": 3.6984305236171915, + "tokens_seen": 870187008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037188565697091275, + "loss": 2.4252, + "theoretical_loss": 3.698403331838873, + "tokens_seen": 870252544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037187562688064193, + "loss": 2.5673, + "theoretical_loss": 3.6983761426815205, + "tokens_seen": 870318080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718655967903711, + "loss": 2.865, + "theoretical_loss": 3.6983489561446827, + "tokens_seen": 870383616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718555667001003, + "loss": 2.7507, + "theoretical_loss": 3.698321772227911, + "tokens_seen": 870449152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037184553660982953, + "loss": 2.5034, + "theoretical_loss": 3.6982945909307547, + "tokens_seen": 870514688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037183550651955866, + "loss": 2.5666, + "theoretical_loss": 3.698267412252765, + "tokens_seen": 870580224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718254764292879, + "loss": 2.7432, + "theoretical_loss": 3.6982402361934916, + "tokens_seen": 870645760 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371815446339017, + "loss": 2.6505, + "theoretical_loss": 3.698213062752486, + "tokens_seen": 870711296 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037180541624874625, + "loss": 2.8117, + "theoretical_loss": 3.698185891929297, + "tokens_seen": 870776832 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037179538615847544, + "loss": 2.7201, + "theoretical_loss": 3.698158723723478, + "tokens_seen": 870842368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717853560682046, + "loss": 2.5757, + "theoretical_loss": 3.6981315581345786, + "tokens_seen": 870907904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717753259779338, + "loss": 2.8679, + "theoretical_loss": 3.698104395162149, + "tokens_seen": 870973440 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371765295887663, + "loss": 2.5399, + "theoretical_loss": 3.6980772348057425, + "tokens_seen": 871038976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037175526579739216, + "loss": 2.7885, + "theoretical_loss": 3.6980500770649085, + "tokens_seen": 871104512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717452357071214, + "loss": 2.707, + "theoretical_loss": 3.6980229219391987, + "tokens_seen": 871170048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717352056168505, + "loss": 2.6914, + "theoretical_loss": 3.6979957694281653, + "tokens_seen": 871235584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037172517552657976, + "loss": 2.7263, + "theoretical_loss": 3.697968619531359, + "tokens_seen": 871301120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717151454363089, + "loss": 2.5678, + "theoretical_loss": 3.697941472248332, + "tokens_seen": 871366656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717051153460381, + "loss": 2.5981, + "theoretical_loss": 3.6979143275786366, + "tokens_seen": 871432192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716950852557673, + "loss": 2.7973, + "theoretical_loss": 3.697887185521824, + "tokens_seen": 871497728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716850551654965, + "loss": 2.897, + "theoretical_loss": 3.6978600460774462, + "tokens_seen": 871563264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 487872, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4374098777770996, + "objective/train/theoretical_loss": 3.6978329092450557, + "objective/train/tokens_used": 892088800, + "theoretical_loss": 3.6978329092450557, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037167502507522566, + "loss": 2.6415, + "theoretical_loss": 3.6978329092450557, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716649949849549, + "loss": 2.8264, + "theoretical_loss": 3.6978057750242046, + "tokens_seen": 871694336 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371654964894684, + "loss": 2.8088, + "theoretical_loss": 3.697778643414445, + "tokens_seen": 871759872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037164493480441326, + "loss": 2.6045, + "theoretical_loss": 3.69775151441533, + "tokens_seen": 871825408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716349047141424, + "loss": 2.806, + "theoretical_loss": 3.697724388026412, + "tokens_seen": 871890944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716248746238716, + "loss": 2.804, + "theoretical_loss": 3.6976972642472425, + "tokens_seen": 871956480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037161484453360086, + "loss": 2.7078, + "theoretical_loss": 3.697670143077376, + "tokens_seen": 872022016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037160481444333, + "loss": 2.6623, + "theoretical_loss": 3.697643024516365, + "tokens_seen": 872087552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715947843530592, + "loss": 2.8471, + "theoretical_loss": 3.697615908563762, + "tokens_seen": 872153088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037158475426278835, + "loss": 2.5388, + "theoretical_loss": 3.6975887952191204, + "tokens_seen": 872218624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715747241725176, + "loss": 2.7085, + "theoretical_loss": 3.697561684481993, + "tokens_seen": 872284160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037156469408224676, + "loss": 2.8703, + "theoretical_loss": 3.6975345763519343, + "tokens_seen": 872349696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037155466399197595, + "loss": 2.6017, + "theoretical_loss": 3.697507470828497, + "tokens_seen": 872415232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715446339017051, + "loss": 2.5009, + "theoretical_loss": 3.6974803679112345, + "tokens_seen": 872480768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037153460381143436, + "loss": 2.5021, + "theoretical_loss": 3.697453267599701, + "tokens_seen": 872546304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715245737211635, + "loss": 3.0077, + "theoretical_loss": 3.6974261698934496, + "tokens_seen": 872611840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715145436308927, + "loss": 2.8439, + "theoretical_loss": 3.6973990747920347, + "tokens_seen": 872677376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037150451354062185, + "loss": 2.7113, + "theoretical_loss": 3.69737198229501, + "tokens_seen": 872742912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714944834503511, + "loss": 2.5746, + "theoretical_loss": 3.6973448924019308, + "tokens_seen": 872808448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037148445336008027, + "loss": 2.7075, + "theoretical_loss": 3.6973178051123496, + "tokens_seen": 872873984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037147442326980945, + "loss": 2.5569, + "theoretical_loss": 3.697290720425822, + "tokens_seen": 872939520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037146439317953863, + "loss": 2.6775, + "theoretical_loss": 3.6972636383419015, + "tokens_seen": 873005056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714543630892678, + "loss": 2.7166, + "theoretical_loss": 3.6972365588601432, + "tokens_seen": 873070592 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371444332998997, + "loss": 2.6978, + "theoretical_loss": 3.697209481980102, + "tokens_seen": 873136128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037143430290872623, + "loss": 2.7577, + "theoretical_loss": 3.697182407701332, + "tokens_seen": 873201664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 488898, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9824957847595215, + "objective/train/theoretical_loss": 3.697155336023389, + "objective/train/tokens_used": 893727200, + "theoretical_loss": 3.697155336023389, + "tokens_seen": 873267200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037142427281845535, + "loss": 2.7698, + "theoretical_loss": 3.697155336023389, + "tokens_seen": 873267200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714142427281846, + "loss": 2.9909, + "theoretical_loss": 3.6971282669458274, + "tokens_seen": 873332736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714042126379137, + "loss": 2.651, + "theoretical_loss": 3.6971012004682025, + "tokens_seen": 873398272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037139418254764295, + "loss": 2.7109, + "theoretical_loss": 3.697074136590069, + "tokens_seen": 873463808 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037138415245737213, + "loss": 2.8332, + "theoretical_loss": 3.6970470753109828, + "tokens_seen": 873529344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713741223671013, + "loss": 2.7787, + "theoretical_loss": 3.6970200166304994, + "tokens_seen": 873594880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713640922768305, + "loss": 2.9767, + "theoretical_loss": 3.696992960548174, + "tokens_seen": 873660416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037135406218655973, + "loss": 2.8765, + "theoretical_loss": 3.6969659070635625, + "tokens_seen": 873725952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037134403209628886, + "loss": 2.7108, + "theoretical_loss": 3.696938856176221, + "tokens_seen": 873791488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713340020060181, + "loss": 2.7555, + "theoretical_loss": 3.6969118078857046, + "tokens_seen": 873857024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713239719157472, + "loss": 2.7623, + "theoretical_loss": 3.6968847621915693, + "tokens_seen": 873922560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037131394182547645, + "loss": 2.7329, + "theoretical_loss": 3.696857719093372, + "tokens_seen": 873988096 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037130391173520564, + "loss": 2.6343, + "theoretical_loss": 3.6968306785906684, + "tokens_seen": 874053632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712938816449348, + "loss": 2.5664, + "theoretical_loss": 3.6968036406830147, + "tokens_seen": 874119168 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371283851554664, + "loss": 2.7419, + "theoretical_loss": 3.6967766053699673, + "tokens_seen": 874184704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712738214643932, + "loss": 2.6855, + "theoretical_loss": 3.6967495726510835, + "tokens_seen": 874250240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037126379137412236, + "loss": 2.8358, + "theoretical_loss": 3.6967225425259187, + "tokens_seen": 874315776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712537612838516, + "loss": 2.7927, + "theoretical_loss": 3.6966955149940306, + "tokens_seen": 874381312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712437311935807, + "loss": 2.6235, + "theoretical_loss": 3.696668490054976, + "tokens_seen": 874446848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037123370110330996, + "loss": 2.5521, + "theoretical_loss": 3.6966414677083113, + "tokens_seen": 874512384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712236710130391, + "loss": 2.936, + "theoretical_loss": 3.696614447953594, + "tokens_seen": 874577920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712136409227683, + "loss": 2.5652, + "theoretical_loss": 3.696587430790381, + "tokens_seen": 874643456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712036108324975, + "loss": 2.7471, + "theoretical_loss": 3.6965604162182304, + "tokens_seen": 874708992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711935807422267, + "loss": 2.6699, + "theoretical_loss": 3.696533404236699, + "tokens_seen": 874774528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037118355065195586, + "loss": 2.8556, + "theoretical_loss": 3.6965063948453434, + "tokens_seen": 874840064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 489972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.883964776992798, + "objective/train/theoretical_loss": 3.6964793880437226, + "objective/train/tokens_used": 895365600, + "theoretical_loss": 3.6964793880437226, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711735205616851, + "loss": 2.8779, + "theoretical_loss": 3.6964793880437226, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711634904714142, + "loss": 2.854, + "theoretical_loss": 3.696452383831394, + "tokens_seen": 874971136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037115346038114346, + "loss": 2.901, + "theoretical_loss": 3.6964253822079147, + "tokens_seen": 875036672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711434302908726, + "loss": 3.0085, + "theoretical_loss": 3.696398383172844, + "tokens_seen": 875102208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711334002006018, + "loss": 2.5525, + "theoretical_loss": 3.696371386725739, + "tokens_seen": 875167744 + }, + { + "epoch": 0.27, + "learning_rate": 0.000371123370110331, + "loss": 2.8206, + "theoretical_loss": 3.696344392866158, + "tokens_seen": 875233280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711133400200602, + "loss": 2.9085, + "theoretical_loss": 3.6963174015936593, + "tokens_seen": 875298816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037110330992978937, + "loss": 2.7336, + "theoretical_loss": 3.6962904129078016, + "tokens_seen": 875364352 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037109327983951855, + "loss": 2.5856, + "theoretical_loss": 3.6962634268081427, + "tokens_seen": 875429888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037108324974924773, + "loss": 2.5817, + "theoretical_loss": 3.6962364432942416, + "tokens_seen": 875495424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037107321965897696, + "loss": 2.808, + "theoretical_loss": 3.6962094623656574, + "tokens_seen": 875560960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710631895687061, + "loss": 2.8817, + "theoretical_loss": 3.696182484021948, + "tokens_seen": 875626496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710531594784353, + "loss": 2.7093, + "theoretical_loss": 3.696155508262673, + "tokens_seen": 875692032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037104312938816445, + "loss": 2.671, + "theoretical_loss": 3.6961285350873916, + "tokens_seen": 875757568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710330992978937, + "loss": 2.6938, + "theoretical_loss": 3.6961015644956623, + "tokens_seen": 875823104 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037102306920762287, + "loss": 2.7216, + "theoretical_loss": 3.696074596487045, + "tokens_seen": 875888640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037101303911735205, + "loss": 2.5135, + "theoretical_loss": 3.6960476310610986, + "tokens_seen": 875954176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037100300902708123, + "loss": 2.663, + "theoretical_loss": 3.696020668217382, + "tokens_seen": 876019712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037099297893681047, + "loss": 2.7203, + "theoretical_loss": 3.6959937079554566, + "tokens_seen": 876085248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709829488465396, + "loss": 2.7627, + "theoretical_loss": 3.69596675027488, + "tokens_seen": 876150784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037097291875626883, + "loss": 2.8865, + "theoretical_loss": 3.6959397951752133, + "tokens_seen": 876216320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037096288866599796, + "loss": 2.6588, + "theoretical_loss": 3.695912842656016, + "tokens_seen": 876281856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709528585757272, + "loss": 2.8049, + "theoretical_loss": 3.695885892716848, + "tokens_seen": 876347392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709428284854564, + "loss": 2.7291, + "theoretical_loss": 3.6958589453572697, + "tokens_seen": 876412928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037093279839518555, + "loss": 2.826, + "theoretical_loss": 3.695832000576841, + "tokens_seen": 876478464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 490667, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.061556816101074, + "objective/train/theoretical_loss": 3.6958050583751225, + "objective/train/tokens_used": 897004000, + "theoretical_loss": 3.6958050583751225, + "tokens_seen": 876544000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037092276830491474, + "loss": 2.8808, + "theoretical_loss": 3.6958050583751225, + "tokens_seen": 876544000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709127382146439, + "loss": 2.7035, + "theoretical_loss": 3.6957781187516745, + "tokens_seen": 876609536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709027081243731, + "loss": 2.8415, + "theoretical_loss": 3.6957511817060578, + "tokens_seen": 876675072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037089267803410233, + "loss": 2.6895, + "theoretical_loss": 3.695724247237832, + "tokens_seen": 876740608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037088264794383146, + "loss": 2.6108, + "theoretical_loss": 3.6956973153465595, + "tokens_seen": 876806144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708726178535607, + "loss": 2.8491, + "theoretical_loss": 3.6956703860318, + "tokens_seen": 876871680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037086258776328993, + "loss": 2.8227, + "theoretical_loss": 3.695643459293115, + "tokens_seen": 876937216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037085255767301906, + "loss": 2.6601, + "theoretical_loss": 3.695616535130065, + "tokens_seen": 877002752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708425275827483, + "loss": 2.7217, + "theoretical_loss": 3.6955896135422117, + "tokens_seen": 877068288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708324974924774, + "loss": 2.8116, + "theoretical_loss": 3.6955626945291167, + "tokens_seen": 877133824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037082246740220665, + "loss": 2.6787, + "theoretical_loss": 3.6955357780903406, + "tokens_seen": 877199360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037081243731193584, + "loss": 2.5873, + "theoretical_loss": 3.6955088642254452, + "tokens_seen": 877264896 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370802407221665, + "loss": 2.8596, + "theoretical_loss": 3.6954819529339926, + "tokens_seen": 877330432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707923771313942, + "loss": 2.6279, + "theoretical_loss": 3.695455044215544, + "tokens_seen": 877395968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707823470411234, + "loss": 2.6978, + "theoretical_loss": 3.695428138069661, + "tokens_seen": 877461504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037077231695085256, + "loss": 2.6799, + "theoretical_loss": 3.6954012344959066, + "tokens_seen": 877527040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707622868605818, + "loss": 2.7125, + "theoretical_loss": 3.6953743334938416, + "tokens_seen": 877592576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707522567703109, + "loss": 2.7077, + "theoretical_loss": 3.6953474350630287, + "tokens_seen": 877658112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037074222668004016, + "loss": 2.81, + "theoretical_loss": 3.6953205392030304, + "tokens_seen": 877723648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707321965897693, + "loss": 2.6282, + "theoretical_loss": 3.6952936459134085, + "tokens_seen": 877789184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707221664994985, + "loss": 2.9963, + "theoretical_loss": 3.6952667551937264, + "tokens_seen": 877854720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707121364092277, + "loss": 2.6417, + "theoretical_loss": 3.695239867043546, + "tokens_seen": 877920256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707021063189569, + "loss": 2.889, + "theoretical_loss": 3.6952129814624297, + "tokens_seen": 877985792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037069207622868606, + "loss": 2.7111, + "theoretical_loss": 3.695186098449941, + "tokens_seen": 878051328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706820461384153, + "loss": 2.7687, + "theoretical_loss": 3.6951592180056423, + "tokens_seen": 878116864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 491863, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.594583511352539, + "objective/train/theoretical_loss": 3.6951323401290974, + "objective/train/tokens_used": 898642400, + "theoretical_loss": 3.6951323401290974, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706720160481444, + "loss": 2.6835, + "theoretical_loss": 3.6951323401290974, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037066198595787366, + "loss": 2.8534, + "theoretical_loss": 3.695105464819868, + "tokens_seen": 878247936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706519558676028, + "loss": 2.758, + "theoretical_loss": 3.6950785920775187, + "tokens_seen": 878313472 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370641925777332, + "loss": 2.9426, + "theoretical_loss": 3.6950517219016117, + "tokens_seen": 878379008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706318956870612, + "loss": 2.8975, + "theoretical_loss": 3.695024854291712, + "tokens_seen": 878444544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706218655967904, + "loss": 2.549, + "theoretical_loss": 3.6949979892473817, + "tokens_seen": 878510080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037061183550651957, + "loss": 2.9225, + "theoretical_loss": 3.694971126768185, + "tokens_seen": 878575616 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037060180541624875, + "loss": 2.5897, + "theoretical_loss": 3.6949442668536854, + "tokens_seen": 878641152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037059177532597793, + "loss": 2.8333, + "theoretical_loss": 3.694917409503447, + "tokens_seen": 878706688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037058174523570716, + "loss": 3.032, + "theoretical_loss": 3.6948905547170336, + "tokens_seen": 878772224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705717151454363, + "loss": 2.8523, + "theoretical_loss": 3.6948637024940094, + "tokens_seen": 878837760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705616850551655, + "loss": 2.8115, + "theoretical_loss": 3.694836852833939, + "tokens_seen": 878903296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037055165496489465, + "loss": 2.618, + "theoretical_loss": 3.694810005736386, + "tokens_seen": 878968832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705416248746239, + "loss": 2.7893, + "theoretical_loss": 3.694783161200916, + "tokens_seen": 879034368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037053159478435307, + "loss": 2.8638, + "theoretical_loss": 3.6947563192270914, + "tokens_seen": 879099904 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037052156469408225, + "loss": 2.5287, + "theoretical_loss": 3.6947294798144785, + "tokens_seen": 879165440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037051153460381143, + "loss": 2.7751, + "theoretical_loss": 3.6947026429626417, + "tokens_seen": 879230976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037050150451354067, + "loss": 2.7179, + "theoretical_loss": 3.6946758086711453, + "tokens_seen": 879296512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704914744232698, + "loss": 2.8058, + "theoretical_loss": 3.694648976939555, + "tokens_seen": 879362048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037048144433299903, + "loss": 2.7717, + "theoretical_loss": 3.694622147767435, + "tokens_seen": 879427584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037047141424272816, + "loss": 2.6822, + "theoretical_loss": 3.694595321154351, + "tokens_seen": 879493120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704613841524574, + "loss": 2.5861, + "theoretical_loss": 3.6945684970998682, + "tokens_seen": 879558656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704513540621866, + "loss": 2.6915, + "theoretical_loss": 3.694541675603552, + "tokens_seen": 879624192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037044132397191575, + "loss": 2.7335, + "theoretical_loss": 3.6945148566649677, + "tokens_seen": 879689728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037043129388164494, + "loss": 2.8133, + "theoretical_loss": 3.694488040283681, + "tokens_seen": 879755264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 492132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4907939434051514, + "objective/train/theoretical_loss": 3.6944612264592576, + "objective/train/tokens_used": 900280800, + "theoretical_loss": 3.6944612264592576, + "tokens_seen": 879820800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704212637913741, + "loss": 2.7163, + "theoretical_loss": 3.6944612264592576, + "tokens_seen": 879820800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704112337011033, + "loss": 2.9041, + "theoretical_loss": 3.694434415191263, + "tokens_seen": 879886336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037040120361083253, + "loss": 2.8063, + "theoretical_loss": 3.6944076064792633, + "tokens_seen": 879951872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037039117352056166, + "loss": 2.7982, + "theoretical_loss": 3.6943808003228247, + "tokens_seen": 880017408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703811434302909, + "loss": 2.8089, + "theoretical_loss": 3.694353996721513, + "tokens_seen": 880082944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703711133400201, + "loss": 2.5928, + "theoretical_loss": 3.694327195674894, + "tokens_seen": 880148480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037036108324974926, + "loss": 2.8074, + "theoretical_loss": 3.694300397182535, + "tokens_seen": 880214016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037035105315947844, + "loss": 2.8316, + "theoretical_loss": 3.6942736012440016, + "tokens_seen": 880279552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703410230692076, + "loss": 2.7329, + "theoretical_loss": 3.6942468078588604, + "tokens_seen": 880345088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703309929789368, + "loss": 2.9809, + "theoretical_loss": 3.6942200170266792, + "tokens_seen": 880410624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037032096288866604, + "loss": 2.8692, + "theoretical_loss": 3.694193228747023, + "tokens_seen": 880476160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037031093279839516, + "loss": 2.7912, + "theoretical_loss": 3.6941664430194594, + "tokens_seen": 880541696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703009027081244, + "loss": 2.738, + "theoretical_loss": 3.6941396598435556, + "tokens_seen": 880607232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702908726178535, + "loss": 2.8439, + "theoretical_loss": 3.6941128792188787, + "tokens_seen": 880672768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037028084252758276, + "loss": 2.624, + "theoretical_loss": 3.6940861011449955, + "tokens_seen": 880738304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037027081243731194, + "loss": 2.9418, + "theoretical_loss": 3.694059325621473, + "tokens_seen": 880803840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702607823470411, + "loss": 2.5756, + "theoretical_loss": 3.6940325526478794, + "tokens_seen": 880869376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702507522567703, + "loss": 2.7575, + "theoretical_loss": 3.6940057822237815, + "tokens_seen": 880934912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702407221664995, + "loss": 2.9176, + "theoretical_loss": 3.6939790143487476, + "tokens_seen": 881000448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037023069207622867, + "loss": 2.6867, + "theoretical_loss": 3.6939522490223444, + "tokens_seen": 881065984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702206619859579, + "loss": 2.8547, + "theoretical_loss": 3.6939254862441397, + "tokens_seen": 881131520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037021063189568703, + "loss": 2.504, + "theoretical_loss": 3.6938987260137024, + "tokens_seen": 881197056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037020060180541626, + "loss": 2.8788, + "theoretical_loss": 3.6938719683306003, + "tokens_seen": 881262592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037019057171514544, + "loss": 2.5329, + "theoretical_loss": 3.6938452131944004, + "tokens_seen": 881328128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701805416248746, + "loss": 2.8602, + "theoretical_loss": 3.6938184606046724, + "tokens_seen": 881393664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 493018, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3380496501922607, + "objective/train/theoretical_loss": 3.6937917105609834, + "objective/train/tokens_used": 901919200, + "theoretical_loss": 3.6937917105609834, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701705115346038, + "loss": 2.6497, + "theoretical_loss": 3.6937917105609834, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370160481444333, + "loss": 2.9965, + "theoretical_loss": 3.6937649630629026, + "tokens_seen": 881524736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037015045135406217, + "loss": 2.7815, + "theoretical_loss": 3.6937382181099983, + "tokens_seen": 881590272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701404212637914, + "loss": 2.5983, + "theoretical_loss": 3.6937114757018388, + "tokens_seen": 881655808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037013039117352053, + "loss": 2.9515, + "theoretical_loss": 3.6936847358379934, + "tokens_seen": 881721344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037012036108324977, + "loss": 2.7363, + "theoretical_loss": 3.693657998518031, + "tokens_seen": 881786880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037011033099297895, + "loss": 2.8001, + "theoretical_loss": 3.6936312637415196, + "tokens_seen": 881852416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037010030090270813, + "loss": 2.7913, + "theoretical_loss": 3.6936045315080293, + "tokens_seen": 881917952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037009027081243736, + "loss": 2.5676, + "theoretical_loss": 3.6935778018171286, + "tokens_seen": 881983488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700802407221665, + "loss": 2.8248, + "theoretical_loss": 3.6935510746683873, + "tokens_seen": 882049024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700702106318957, + "loss": 2.7201, + "theoretical_loss": 3.693524350061374, + "tokens_seen": 882114560 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037006018054162485, + "loss": 2.7187, + "theoretical_loss": 3.693497627995659, + "tokens_seen": 882180096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700501504513541, + "loss": 2.9054, + "theoretical_loss": 3.693470908470812, + "tokens_seen": 882245632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037004012036108327, + "loss": 2.4963, + "theoretical_loss": 3.6934441914864014, + "tokens_seen": 882311168 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037003009027081245, + "loss": 2.6246, + "theoretical_loss": 3.693417477041998, + "tokens_seen": 882376704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037002006018054163, + "loss": 2.7817, + "theoretical_loss": 3.693390765137172, + "tokens_seen": 882442240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037001003009027087, + "loss": 2.7273, + "theoretical_loss": 3.693364055771492, + "tokens_seen": 882507776 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037, + "loss": 2.8035, + "theoretical_loss": 3.69333734894453, + "tokens_seen": 882573312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036998996990972923, + "loss": 2.9529, + "theoretical_loss": 3.693310644655855, + "tokens_seen": 882638848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036997993981945836, + "loss": 2.8309, + "theoretical_loss": 3.693283942905037, + "tokens_seen": 882704384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699699097291876, + "loss": 3.0268, + "theoretical_loss": 3.6932572436916473, + "tokens_seen": 882769920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699598796389168, + "loss": 2.9085, + "theoretical_loss": 3.6932305470152564, + "tokens_seen": 882835456 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036994984954864595, + "loss": 2.8015, + "theoretical_loss": 3.693203852875434, + "tokens_seen": 882900992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036993981945837514, + "loss": 2.5191, + "theoretical_loss": 3.693177161271752, + "tokens_seen": 882966528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699297893681043, + "loss": 2.8193, + "theoretical_loss": 3.6931504722037802, + "tokens_seen": 883032064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 493587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.08097767829895, + "objective/train/theoretical_loss": 3.6931237856710903, + "objective/train/tokens_used": 903557600, + "theoretical_loss": 3.6931237856710903, + "tokens_seen": 883097600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699197592778335, + "loss": 2.993, + "theoretical_loss": 3.6931237856710903, + "tokens_seen": 883097600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036990972918756273, + "loss": 2.6583, + "theoretical_loss": 3.693097101673253, + "tokens_seen": 883163136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036989969909729186, + "loss": 2.7407, + "theoretical_loss": 3.6930704202098394, + "tokens_seen": 883228672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698896690070211, + "loss": 2.965, + "theoretical_loss": 3.6930437412804213, + "tokens_seen": 883294208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698796389167503, + "loss": 2.8227, + "theoretical_loss": 3.693017064884569, + "tokens_seen": 883359744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036986960882647946, + "loss": 2.8113, + "theoretical_loss": 3.692990391021855, + "tokens_seen": 883425280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036985957873620864, + "loss": 2.77, + "theoretical_loss": 3.69296371969185, + "tokens_seen": 883490816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698495486459378, + "loss": 2.7267, + "theoretical_loss": 3.692937050894127, + "tokens_seen": 883556352 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369839518555667, + "loss": 2.5705, + "theoretical_loss": 3.692910384628256, + "tokens_seen": 883621888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036982948846539624, + "loss": 2.9653, + "theoretical_loss": 3.69288372089381, + "tokens_seen": 883687424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036981945837512536, + "loss": 2.7199, + "theoretical_loss": 3.6928570596903607, + "tokens_seen": 883752960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698094282848546, + "loss": 2.7355, + "theoretical_loss": 3.6928304010174804, + "tokens_seen": 883818496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697993981945837, + "loss": 2.5483, + "theoretical_loss": 3.692803744874741, + "tokens_seen": 883884032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036978936810431296, + "loss": 2.8324, + "theoretical_loss": 3.692777091261715, + "tokens_seen": 883949568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036977933801404214, + "loss": 2.5926, + "theoretical_loss": 3.692750440177975, + "tokens_seen": 884015104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697693079237713, + "loss": 2.809, + "theoretical_loss": 3.6927237916230933, + "tokens_seen": 884080640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697592778335005, + "loss": 2.8743, + "theoretical_loss": 3.692697145596642, + "tokens_seen": 884146176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697492477432297, + "loss": 2.6491, + "theoretical_loss": 3.6926705020981943, + "tokens_seen": 884211712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036973921765295887, + "loss": 2.8035, + "theoretical_loss": 3.692643861127323, + "tokens_seen": 884277248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697291875626881, + "loss": 2.9265, + "theoretical_loss": 3.692617222683601, + "tokens_seen": 884342784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036971915747241723, + "loss": 2.7762, + "theoretical_loss": 3.6925905867666016, + "tokens_seen": 884408320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036970912738214646, + "loss": 2.8945, + "theoretical_loss": 3.6925639533758976, + "tokens_seen": 884473856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036969909729187565, + "loss": 2.9008, + "theoretical_loss": 3.692537322511062, + "tokens_seen": 884539392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696890672016048, + "loss": 2.7927, + "theoretical_loss": 3.6925106941716686, + "tokens_seen": 884604928 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369679037111334, + "loss": 2.5308, + "theoretical_loss": 3.692484068357291, + "tokens_seen": 884670464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 494919, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.518890857696533, + "objective/train/theoretical_loss": 3.692457445067501, + "objective/train/tokens_used": 905196000, + "theoretical_loss": 3.692457445067501, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696690070210632, + "loss": 2.6387, + "theoretical_loss": 3.692457445067501, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036965897693079237, + "loss": 2.8342, + "theoretical_loss": 3.692430824301875, + "tokens_seen": 884801536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696489468405216, + "loss": 2.6288, + "theoretical_loss": 3.692404206059985, + "tokens_seen": 884867072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036963891675025073, + "loss": 2.7104, + "theoretical_loss": 3.692377590341405, + "tokens_seen": 884932608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036962888665997997, + "loss": 2.8244, + "theoretical_loss": 3.6923509771457095, + "tokens_seen": 884998144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696188565697091, + "loss": 2.8814, + "theoretical_loss": 3.692324366472472, + "tokens_seen": 885063680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036960882647943833, + "loss": 2.9496, + "theoretical_loss": 3.692297758321267, + "tokens_seen": 885129216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695987963891675, + "loss": 2.5732, + "theoretical_loss": 3.692271152691669, + "tokens_seen": 885194752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695887662988967, + "loss": 2.6226, + "theoretical_loss": 3.6922445495832514, + "tokens_seen": 885260288 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036957873620862587, + "loss": 2.8412, + "theoretical_loss": 3.69221794899559, + "tokens_seen": 885325824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036956870611835505, + "loss": 2.6286, + "theoretical_loss": 3.6921913509282582, + "tokens_seen": 885391360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036955867602808424, + "loss": 2.7941, + "theoretical_loss": 3.692164755380831, + "tokens_seen": 885456896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036954864593781347, + "loss": 2.7921, + "theoretical_loss": 3.692138162352884, + "tokens_seen": 885522432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695386158475426, + "loss": 2.793, + "theoretical_loss": 3.6921115718439905, + "tokens_seen": 885587968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036952858575727183, + "loss": 2.7073, + "theoretical_loss": 3.6920849838537273, + "tokens_seen": 885653504 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369518555667001, + "loss": 2.8973, + "theoretical_loss": 3.6920583983816684, + "tokens_seen": 885719040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695085255767302, + "loss": 2.797, + "theoretical_loss": 3.6920318154273892, + "tokens_seen": 885784576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694984954864594, + "loss": 2.8963, + "theoretical_loss": 3.6920052349904653, + "tokens_seen": 885850112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036948846539618856, + "loss": 2.6427, + "theoretical_loss": 3.691978657070471, + "tokens_seen": 885915648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036947843530591774, + "loss": 2.7269, + "theoretical_loss": 3.691952081666983, + "tokens_seen": 885981184 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369468405215647, + "loss": 2.745, + "theoretical_loss": 3.691925508779577, + "tokens_seen": 886046720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694583751253761, + "loss": 2.5784, + "theoretical_loss": 3.691898938407828, + "tokens_seen": 886112256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036944834503510534, + "loss": 2.7445, + "theoretical_loss": 3.691872370551312, + "tokens_seen": 886177792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036943831494483446, + "loss": 2.6901, + "theoretical_loss": 3.6918458052096046, + "tokens_seen": 886243328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694282848545637, + "loss": 2.8246, + "theoretical_loss": 3.691819242382283, + "tokens_seen": 886308864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 496242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7018520832061768, + "objective/train/theoretical_loss": 3.6917926820689217, + "objective/train/tokens_used": 906834400, + "theoretical_loss": 3.6917926820689217, + "tokens_seen": 886374400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694182547642929, + "loss": 2.6236, + "theoretical_loss": 3.6917926820689217, + "tokens_seen": 886374400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036940822467402206, + "loss": 2.6371, + "theoretical_loss": 3.6917661242690984, + "tokens_seen": 886439936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036939819458375124, + "loss": 2.8313, + "theoretical_loss": 3.691739568982388, + "tokens_seen": 886505472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693881644934805, + "loss": 2.8547, + "theoretical_loss": 3.691713016208368, + "tokens_seen": 886571008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693781344032096, + "loss": 2.9052, + "theoretical_loss": 3.691686465946615, + "tokens_seen": 886636544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036936810431293884, + "loss": 2.8262, + "theoretical_loss": 3.691659918196705, + "tokens_seen": 886702080 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369358074222668, + "loss": 2.5036, + "theoretical_loss": 3.691633372958215, + "tokens_seen": 886767616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693480441323972, + "loss": 2.7461, + "theoretical_loss": 3.6916068302307217, + "tokens_seen": 886833152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036933801404212644, + "loss": 2.6719, + "theoretical_loss": 3.6915802900138024, + "tokens_seen": 886898688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036932798395185556, + "loss": 2.8203, + "theoretical_loss": 3.691553752307034, + "tokens_seen": 886964224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693179538615848, + "loss": 2.7727, + "theoretical_loss": 3.6915272171099938, + "tokens_seen": 887029760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693079237713139, + "loss": 2.5439, + "theoretical_loss": 3.691500684422258, + "tokens_seen": 887095296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036929789368104316, + "loss": 2.6267, + "theoretical_loss": 3.691474154243405, + "tokens_seen": 887160832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036928786359077234, + "loss": 2.8903, + "theoretical_loss": 3.6914476265730123, + "tokens_seen": 887226368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692778335005015, + "loss": 2.8947, + "theoretical_loss": 3.6914211014106573, + "tokens_seen": 887291904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692678034102307, + "loss": 3.0139, + "theoretical_loss": 3.6913945787559177, + "tokens_seen": 887357440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692577733199599, + "loss": 2.8974, + "theoretical_loss": 3.691368058608371, + "tokens_seen": 887422976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036924774322968907, + "loss": 2.7185, + "theoretical_loss": 3.6913415409675947, + "tokens_seen": 887488512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692377131394183, + "loss": 2.7825, + "theoretical_loss": 3.691315025833168, + "tokens_seen": 887554048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036922768304914743, + "loss": 2.737, + "theoretical_loss": 3.6912885132046673, + "tokens_seen": 887619584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036921765295887666, + "loss": 2.7198, + "theoretical_loss": 3.6912620030816723, + "tokens_seen": 887685120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036920762286860585, + "loss": 2.7995, + "theoretical_loss": 3.6912354954637605, + "tokens_seen": 887750656 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369197592778335, + "loss": 2.732, + "theoretical_loss": 3.6912089903505105, + "tokens_seen": 887816192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691875626880642, + "loss": 2.8471, + "theoretical_loss": 3.691182487741501, + "tokens_seen": 887881728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691775325977934, + "loss": 2.6629, + "theoretical_loss": 3.69115598763631, + "tokens_seen": 887947264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 496686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.581552267074585, + "objective/train/theoretical_loss": 3.6911294900345166, + "objective/train/tokens_used": 908472800, + "theoretical_loss": 3.6911294900345166, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036916750250752257, + "loss": 2.5457, + "theoretical_loss": 3.6911294900345166, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691574724172518, + "loss": 2.9441, + "theoretical_loss": 3.691102994935699, + "tokens_seen": 888078336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036914744232698093, + "loss": 2.9934, + "theoretical_loss": 3.6910765023394374, + "tokens_seen": 888143872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036913741223671017, + "loss": 2.813, + "theoretical_loss": 3.6910500122453094, + "tokens_seen": 888209408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691273821464393, + "loss": 2.7331, + "theoretical_loss": 3.691023524652895, + "tokens_seen": 888274944 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036911735205616853, + "loss": 2.5818, + "theoretical_loss": 3.6909970395617724, + "tokens_seen": 888340480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691073219658977, + "loss": 2.7829, + "theoretical_loss": 3.690970556971522, + "tokens_seen": 888406016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690972918756269, + "loss": 2.603, + "theoretical_loss": 3.6909440768817223, + "tokens_seen": 888471552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690872617853561, + "loss": 2.7331, + "theoretical_loss": 3.6909175992919536, + "tokens_seen": 888537088 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036907723169508525, + "loss": 2.5585, + "theoretical_loss": 3.690891124201795, + "tokens_seen": 888602624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036906720160481444, + "loss": 2.7386, + "theoretical_loss": 3.690864651610826, + "tokens_seen": 888668160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036905717151454367, + "loss": 2.7889, + "theoretical_loss": 3.690838181518627, + "tokens_seen": 888733696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690471414242728, + "loss": 2.9249, + "theoretical_loss": 3.690811713924778, + "tokens_seen": 888799232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036903711133400203, + "loss": 2.835, + "theoretical_loss": 3.690785248828858, + "tokens_seen": 888864768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690270812437312, + "loss": 2.5885, + "theoretical_loss": 3.690758786230448, + "tokens_seen": 888930304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690170511534604, + "loss": 2.8022, + "theoretical_loss": 3.690732326129128, + "tokens_seen": 888995840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690070210631896, + "loss": 2.7309, + "theoretical_loss": 3.690705868524478, + "tokens_seen": 889061376 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036899699097291876, + "loss": 2.8972, + "theoretical_loss": 3.6906794134160785, + "tokens_seen": 889126912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036898696088264794, + "loss": 2.7127, + "theoretical_loss": 3.6906529608035106, + "tokens_seen": 889192448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689769307923772, + "loss": 2.949, + "theoretical_loss": 3.6906265106863545, + "tokens_seen": 889257984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689669007021063, + "loss": 2.6056, + "theoretical_loss": 3.690600063064191, + "tokens_seen": 889323520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036895687061183554, + "loss": 2.7401, + "theoretical_loss": 3.6905736179366, + "tokens_seen": 889389056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036894684052156466, + "loss": 2.8579, + "theoretical_loss": 3.690547175303164, + "tokens_seen": 889454592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689368104312939, + "loss": 2.7001, + "theoretical_loss": 3.690520735163463, + "tokens_seen": 889520128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689267803410231, + "loss": 2.6667, + "theoretical_loss": 3.6904942975170787, + "tokens_seen": 889585664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 497720, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7590861320495605, + "objective/train/theoretical_loss": 3.6904678623635916, + "objective/train/tokens_used": 910111200, + "theoretical_loss": 3.6904678623635916, + "tokens_seen": 889651200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036891675025075226, + "loss": 2.8205, + "theoretical_loss": 3.6904678623635916, + "tokens_seen": 889651200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036890672016048144, + "loss": 2.5425, + "theoretical_loss": 3.6904414297025836, + "tokens_seen": 889716736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688966900702107, + "loss": 2.9111, + "theoretical_loss": 3.690414999533636, + "tokens_seen": 889782272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688866599799398, + "loss": 2.8207, + "theoretical_loss": 3.69038857185633, + "tokens_seen": 889847808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036887662988966904, + "loss": 2.5862, + "theoretical_loss": 3.6903621466702474, + "tokens_seen": 889913344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036886659979939817, + "loss": 2.7536, + "theoretical_loss": 3.6903357239749703, + "tokens_seen": 889978880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688565697091274, + "loss": 2.7551, + "theoretical_loss": 3.6903093037700803, + "tokens_seen": 890044416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688465396188566, + "loss": 2.7463, + "theoretical_loss": 3.690282886055159, + "tokens_seen": 890109952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036883650952858576, + "loss": 2.6848, + "theoretical_loss": 3.690256470829789, + "tokens_seen": 890175488 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036882647943831494, + "loss": 2.8369, + "theoretical_loss": 3.690230058093552, + "tokens_seen": 890241024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688164493480441, + "loss": 2.7004, + "theoretical_loss": 3.6902036478460305, + "tokens_seen": 890306560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688064192577733, + "loss": 2.6664, + "theoretical_loss": 3.6901772400868063, + "tokens_seen": 890372096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036879638916750254, + "loss": 2.6107, + "theoretical_loss": 3.6901508348154626, + "tokens_seen": 890437632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036878635907723167, + "loss": 2.6259, + "theoretical_loss": 3.6901244320315816, + "tokens_seen": 890503168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687763289869609, + "loss": 2.6725, + "theoretical_loss": 3.6900980317347454, + "tokens_seen": 890568704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036876629889669003, + "loss": 2.6895, + "theoretical_loss": 3.6900716339245383, + "tokens_seen": 890634240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036875626880641927, + "loss": 2.7007, + "theoretical_loss": 3.6900452386005407, + "tokens_seen": 890699776 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036874623871614845, + "loss": 2.7271, + "theoretical_loss": 3.6900188457623377, + "tokens_seen": 890765312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036873620862587763, + "loss": 2.7241, + "theoretical_loss": 3.689992455409512, + "tokens_seen": 890830848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687261785356068, + "loss": 2.5983, + "theoretical_loss": 3.6899660675416457, + "tokens_seen": 890896384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036871614844533605, + "loss": 2.9601, + "theoretical_loss": 3.6899396821583226, + "tokens_seen": 890961920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036870611835506517, + "loss": 2.7535, + "theoretical_loss": 3.6899132992591266, + "tokens_seen": 891027456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686960882647944, + "loss": 2.5921, + "theoretical_loss": 3.68988691884364, + "tokens_seen": 891092992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036868605817452353, + "loss": 2.887, + "theoretical_loss": 3.6898605409114476, + "tokens_seen": 891158528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036867602808425277, + "loss": 2.6801, + "theoretical_loss": 3.6898341654621314, + "tokens_seen": 891224064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 498750, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7770748138427734, + "objective/train/theoretical_loss": 3.6898077924952775, + "objective/train/tokens_used": 911749600, + "theoretical_loss": 3.6898077924952775, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036866599799398195, + "loss": 2.663, + "theoretical_loss": 3.6898077924952775, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036865596790371113, + "loss": 2.883, + "theoretical_loss": 3.6897814220104674, + "tokens_seen": 891355136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686459378134403, + "loss": 2.7317, + "theoretical_loss": 3.689755054007286, + "tokens_seen": 891420672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686359077231695, + "loss": 3.0149, + "theoretical_loss": 3.6897286884853173, + "tokens_seen": 891486208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686258776328987, + "loss": 2.7615, + "theoretical_loss": 3.6897023254441463, + "tokens_seen": 891551744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686158475426279, + "loss": 2.6727, + "theoretical_loss": 3.689675964883356, + "tokens_seen": 891617280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686058174523571, + "loss": 2.8897, + "theoretical_loss": 3.689649606802531, + "tokens_seen": 891682816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685957873620863, + "loss": 2.7298, + "theoretical_loss": 3.6896232512012563, + "tokens_seen": 891748352 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036858575727181545, + "loss": 2.6903, + "theoretical_loss": 3.6895968980791163, + "tokens_seen": 891813888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036857572718154464, + "loss": 2.8455, + "theoretical_loss": 3.689570547435695, + "tokens_seen": 891879424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036856569709127387, + "loss": 2.9195, + "theoretical_loss": 3.6895441992705775, + "tokens_seen": 891944960 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368555667001003, + "loss": 2.9814, + "theoretical_loss": 3.689517853583349, + "tokens_seen": 892010496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036854563691073223, + "loss": 2.5849, + "theoretical_loss": 3.6894915103735944, + "tokens_seen": 892076032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685356068204614, + "loss": 2.847, + "theoretical_loss": 3.689465169640898, + "tokens_seen": 892141568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685255767301906, + "loss": 2.8258, + "theoretical_loss": 3.6894388313848454, + "tokens_seen": 892207104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685155466399198, + "loss": 2.6168, + "theoretical_loss": 3.6894124956050223, + "tokens_seen": 892272640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036850551654964896, + "loss": 2.7964, + "theoretical_loss": 3.6893861623010133, + "tokens_seen": 892338176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036849548645937814, + "loss": 2.6983, + "theoretical_loss": 3.6893598314724043, + "tokens_seen": 892403712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684854563691074, + "loss": 2.502, + "theoretical_loss": 3.689333503118781, + "tokens_seen": 892469248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684754262788365, + "loss": 2.8063, + "theoretical_loss": 3.6893071772397286, + "tokens_seen": 892534784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036846539618856574, + "loss": 2.6416, + "theoretical_loss": 3.689280853834833, + "tokens_seen": 892600320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036845536609829486, + "loss": 2.7694, + "theoretical_loss": 3.6892545329036803, + "tokens_seen": 892665856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684453360080241, + "loss": 2.3705, + "theoretical_loss": 3.6892282144458557, + "tokens_seen": 892731392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684353059177533, + "loss": 2.7649, + "theoretical_loss": 3.6892018984609463, + "tokens_seen": 892796928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036842527582748246, + "loss": 2.8509, + "theoretical_loss": 3.6891755849485373, + "tokens_seen": 892862464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 499509, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5587472915649414, + "objective/train/theoretical_loss": 3.6891492739082157, + "objective/train/tokens_used": 913388000, + "theoretical_loss": 3.6891492739082157, + "tokens_seen": 892928000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036841524573721164, + "loss": 2.8054, + "theoretical_loss": 3.6891492739082157, + "tokens_seen": 892928000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684052156469409, + "loss": 2.5245, + "theoretical_loss": 3.6891229653395667, + "tokens_seen": 892993536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036839518555667, + "loss": 2.39, + "theoretical_loss": 3.6890966592421783, + "tokens_seen": 893059072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036838515546639924, + "loss": 2.5772, + "theoretical_loss": 3.689070355615636, + "tokens_seen": 893124608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036837512537612837, + "loss": 2.8144, + "theoretical_loss": 3.6890440544595267, + "tokens_seen": 893190144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683650952858576, + "loss": 2.7312, + "theoretical_loss": 3.6890177557734374, + "tokens_seen": 893255680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683550651955868, + "loss": 2.8735, + "theoretical_loss": 3.6889914595569544, + "tokens_seen": 893321216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036834503510531596, + "loss": 2.6178, + "theoretical_loss": 3.688965165809665, + "tokens_seen": 893386752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036833500501504514, + "loss": 2.7219, + "theoretical_loss": 3.688938874531156, + "tokens_seen": 893452288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683249749247743, + "loss": 2.7928, + "theoretical_loss": 3.688912585721015, + "tokens_seen": 893517824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683149448345035, + "loss": 2.6863, + "theoretical_loss": 3.6888862993788285, + "tokens_seen": 893583360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036830491474423274, + "loss": 2.8111, + "theoretical_loss": 3.6888600155041846, + "tokens_seen": 893648896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036829488465396187, + "loss": 2.7794, + "theoretical_loss": 3.6888337340966704, + "tokens_seen": 893714432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682848545636911, + "loss": 2.7854, + "theoretical_loss": 3.688807455155873, + "tokens_seen": 893779968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036827482447342023, + "loss": 2.8295, + "theoretical_loss": 3.688781178681381, + "tokens_seen": 893845504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036826479438314947, + "loss": 2.7785, + "theoretical_loss": 3.6887549046727814, + "tokens_seen": 893911040 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036825476429287865, + "loss": 2.7202, + "theoretical_loss": 3.6887286331296627, + "tokens_seen": 893976576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036824473420260783, + "loss": 2.6292, + "theoretical_loss": 3.6887023640516117, + "tokens_seen": 894042112 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368234704112337, + "loss": 2.5048, + "theoretical_loss": 3.688676097438217, + "tokens_seen": 894107648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036822467402206625, + "loss": 2.7371, + "theoretical_loss": 3.6886498332890674, + "tokens_seen": 894173184 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036821464393179537, + "loss": 2.9452, + "theoretical_loss": 3.68862357160375, + "tokens_seen": 894238720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682046138415246, + "loss": 2.7559, + "theoretical_loss": 3.6885973123818543, + "tokens_seen": 894304256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036819458375125373, + "loss": 2.7215, + "theoretical_loss": 3.688571055622967, + "tokens_seen": 894369792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036818455366098297, + "loss": 2.6896, + "theoretical_loss": 3.688544801326679, + "tokens_seen": 894435328 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036817452357071215, + "loss": 2.7963, + "theoretical_loss": 3.688518549492577, + "tokens_seen": 894500864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 500468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6406912803649902, + "objective/train/theoretical_loss": 3.6884923001202505, + "objective/train/tokens_used": 915026400, + "theoretical_loss": 3.6884923001202505, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036816449348044133, + "loss": 2.8942, + "theoretical_loss": 3.6884923001202505, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681544633901705, + "loss": 2.6235, + "theoretical_loss": 3.6884660532092886, + "tokens_seen": 894631936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681444332998997, + "loss": 2.8042, + "theoretical_loss": 3.6884398087592793, + "tokens_seen": 894697472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681344032096289, + "loss": 2.82, + "theoretical_loss": 3.688413566769812, + "tokens_seen": 894763008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681243731193581, + "loss": 2.9533, + "theoretical_loss": 3.6883873272404766, + "tokens_seen": 894828544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036811434302908724, + "loss": 2.6947, + "theoretical_loss": 3.6883610901708614, + "tokens_seen": 894894080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681043129388165, + "loss": 2.8405, + "theoretical_loss": 3.688334855560556, + "tokens_seen": 894959616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680942828485456, + "loss": 2.8924, + "theoretical_loss": 3.6883086234091493, + "tokens_seen": 895025152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036808425275827484, + "loss": 2.7066, + "theoretical_loss": 3.688282393716232, + "tokens_seen": 895090688 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368074222668004, + "loss": 2.8829, + "theoretical_loss": 3.688256166481393, + "tokens_seen": 895156224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680641925777332, + "loss": 2.8899, + "theoretical_loss": 3.688229941704222, + "tokens_seen": 895221760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680541624874624, + "loss": 2.8019, + "theoretical_loss": 3.6882037193843082, + "tokens_seen": 895287296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680441323971916, + "loss": 2.4944, + "theoretical_loss": 3.6881774995212426, + "tokens_seen": 895352832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036803410230692074, + "loss": 2.4534, + "theoretical_loss": 3.6881512821146147, + "tokens_seen": 895418368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036802407221665, + "loss": 2.7225, + "theoretical_loss": 3.688125067164015, + "tokens_seen": 895483904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680140421263791, + "loss": 2.8461, + "theoretical_loss": 3.6880988546690325, + "tokens_seen": 895549440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036800401203610834, + "loss": 3.0693, + "theoretical_loss": 3.6880726446292593, + "tokens_seen": 895614976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679939819458375, + "loss": 2.7929, + "theoretical_loss": 3.6880464370442843, + "tokens_seen": 895680512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679839518555667, + "loss": 2.7852, + "theoretical_loss": 3.6880202319136988, + "tokens_seen": 895746048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679739217652959, + "loss": 2.7983, + "theoretical_loss": 3.6879940292370934, + "tokens_seen": 895811584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036796389167502506, + "loss": 2.7209, + "theoretical_loss": 3.687967829014058, + "tokens_seen": 895877120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036795386158475424, + "loss": 2.8093, + "theoretical_loss": 3.6879416312441844, + "tokens_seen": 895942656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679438314944835, + "loss": 2.8308, + "theoretical_loss": 3.6879154359270627, + "tokens_seen": 896008192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679338014042126, + "loss": 2.6901, + "theoretical_loss": 3.6878892430622843, + "tokens_seen": 896073728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036792377131394184, + "loss": 2.8194, + "theoretical_loss": 3.6878630526494405, + "tokens_seen": 896139264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 501526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.16762113571167, + "objective/train/theoretical_loss": 3.687836864688122, + "objective/train/tokens_used": 916664800, + "theoretical_loss": 3.687836864688122, + "tokens_seen": 896204800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036791374122367097, + "loss": 3.0439, + "theoretical_loss": 3.687836864688122, + "tokens_seen": 896204800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679037111334002, + "loss": 2.8419, + "theoretical_loss": 3.68781067917792, + "tokens_seen": 896270336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678936810431294, + "loss": 3.0514, + "theoretical_loss": 3.687784496118426, + "tokens_seen": 896335872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036788365095285857, + "loss": 2.8178, + "theoretical_loss": 3.687758315509232, + "tokens_seen": 896401408 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036787362086258775, + "loss": 2.7582, + "theoretical_loss": 3.687732137349929, + "tokens_seen": 896466944 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367863590772317, + "loss": 3.0367, + "theoretical_loss": 3.687705961640109, + "tokens_seen": 896532480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036785356068204616, + "loss": 2.7561, + "theoretical_loss": 3.687679788379364, + "tokens_seen": 896598016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036784353059177535, + "loss": 2.6126, + "theoretical_loss": 3.6876536175672854, + "tokens_seen": 896663552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678335005015045, + "loss": 2.9616, + "theoretical_loss": 3.687627449203465, + "tokens_seen": 896729088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678234704112337, + "loss": 2.7894, + "theoretical_loss": 3.6876012832874956, + "tokens_seen": 896794624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036781344032096294, + "loss": 2.7923, + "theoretical_loss": 3.6875751198189683, + "tokens_seen": 896860160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036780341023069207, + "loss": 2.7743, + "theoretical_loss": 3.687548958797476, + "tokens_seen": 896925696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677933801404213, + "loss": 2.873, + "theoretical_loss": 3.6875228002226113, + "tokens_seen": 896991232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036778335005015043, + "loss": 2.7466, + "theoretical_loss": 3.6874966440939665, + "tokens_seen": 897056768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036777331995987967, + "loss": 2.9035, + "theoretical_loss": 3.6874704904111337, + "tokens_seen": 897122304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036776328986960885, + "loss": 2.9417, + "theoretical_loss": 3.687444339173706, + "tokens_seen": 897187840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036775325977933803, + "loss": 2.7169, + "theoretical_loss": 3.687418190381276, + "tokens_seen": 897253376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677432296890672, + "loss": 2.6652, + "theoretical_loss": 3.6873920440334373, + "tokens_seen": 897318912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036773319959879645, + "loss": 2.402, + "theoretical_loss": 3.687365900129781, + "tokens_seen": 897384448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036772316950852557, + "loss": 2.8106, + "theoretical_loss": 3.6873397586699026, + "tokens_seen": 897449984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677131394182548, + "loss": 2.7959, + "theoretical_loss": 3.687313619653393, + "tokens_seen": 897515520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036770310932798393, + "loss": 2.6755, + "theoretical_loss": 3.6872874830798463, + "tokens_seen": 897581056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036769307923771317, + "loss": 2.8384, + "theoretical_loss": 3.6872613489488564, + "tokens_seen": 897646592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036768304914744235, + "loss": 2.7293, + "theoretical_loss": 3.6872352172600156, + "tokens_seen": 897712128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036767301905717153, + "loss": 2.5484, + "theoretical_loss": 3.6872090880129185, + "tokens_seen": 897777664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 502221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.841933012008667, + "objective/train/theoretical_loss": 3.6871829612071583, + "objective/train/tokens_used": 918303200, + "theoretical_loss": 3.6871829612071583, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676629889669007, + "loss": 2.6498, + "theoretical_loss": 3.6871829612071583, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676529588766299, + "loss": 2.8582, + "theoretical_loss": 3.6871568368423286, + "tokens_seen": 897908736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676429287863591, + "loss": 2.7102, + "theoretical_loss": 3.6871307149180232, + "tokens_seen": 897974272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676328986960883, + "loss": 2.7657, + "theoretical_loss": 3.687104595433836, + "tokens_seen": 898039808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036762286860581744, + "loss": 2.7004, + "theoretical_loss": 3.6870784783893615, + "tokens_seen": 898105344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676128385155467, + "loss": 2.7002, + "theoretical_loss": 3.6870523637841934, + "tokens_seen": 898170880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676028084252758, + "loss": 2.6307, + "theoretical_loss": 3.687026251617926, + "tokens_seen": 898236416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036759277833500504, + "loss": 2.877, + "theoretical_loss": 3.6870001418901532, + "tokens_seen": 898301952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675827482447342, + "loss": 2.9289, + "theoretical_loss": 3.6869740346004702, + "tokens_seen": 898367488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675727181544634, + "loss": 2.7154, + "theoretical_loss": 3.686947929748471, + "tokens_seen": 898433024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675626880641926, + "loss": 3.0352, + "theoretical_loss": 3.6869218273337503, + "tokens_seen": 898498560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675526579739218, + "loss": 2.8265, + "theoretical_loss": 3.6868957273559024, + "tokens_seen": 898564096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036754262788365094, + "loss": 2.6395, + "theoretical_loss": 3.686869629814523, + "tokens_seen": 898629632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675325977933802, + "loss": 2.8067, + "theoretical_loss": 3.6868435347092063, + "tokens_seen": 898695168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675225677031093, + "loss": 2.7041, + "theoretical_loss": 3.686817442039547, + "tokens_seen": 898760704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036751253761283854, + "loss": 2.8153, + "theoretical_loss": 3.6867913518051414, + "tokens_seen": 898826240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675025075225677, + "loss": 2.6424, + "theoretical_loss": 3.6867652640055835, + "tokens_seen": 898891776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674924774322969, + "loss": 2.8134, + "theoretical_loss": 3.686739178640469, + "tokens_seen": 898957312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674824473420261, + "loss": 2.6906, + "theoretical_loss": 3.6867130957093934, + "tokens_seen": 899022848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036747241725175526, + "loss": 2.707, + "theoretical_loss": 3.6866870152119517, + "tokens_seen": 899088384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036746238716148444, + "loss": 2.9336, + "theoretical_loss": 3.6866609371477397, + "tokens_seen": 899153920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674523570712137, + "loss": 2.5698, + "theoretical_loss": 3.6866348615163536, + "tokens_seen": 899219456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674423269809428, + "loss": 2.7786, + "theoretical_loss": 3.686608788317389, + "tokens_seen": 899284992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036743229689067204, + "loss": 2.795, + "theoretical_loss": 3.686582717550441, + "tokens_seen": 899350528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036742226680040117, + "loss": 2.8089, + "theoretical_loss": 3.6865566492151056, + "tokens_seen": 899416064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 503170, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.086702585220337, + "objective/train/theoretical_loss": 3.6865305833109794, + "objective/train/tokens_used": 919941600, + "theoretical_loss": 3.6865305833109794, + "tokens_seen": 899481600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674122367101304, + "loss": 2.8526, + "theoretical_loss": 3.6865305833109794, + "tokens_seen": 899481600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674022066198596, + "loss": 2.7103, + "theoretical_loss": 3.6865045198376594, + "tokens_seen": 899547136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036739217652958877, + "loss": 2.5794, + "theoretical_loss": 3.6864784587947397, + "tokens_seen": 899612672 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036738214643931795, + "loss": 2.8274, + "theoretical_loss": 3.6864524001818184, + "tokens_seen": 899678208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673721163490472, + "loss": 2.7288, + "theoretical_loss": 3.686426343998491, + "tokens_seen": 899743744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673620862587763, + "loss": 2.7591, + "theoretical_loss": 3.6864002902443547, + "tokens_seen": 899809280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036735205616850555, + "loss": 2.5409, + "theoretical_loss": 3.6863742389190053, + "tokens_seen": 899874816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036734202607823467, + "loss": 2.87, + "theoretical_loss": 3.6863481900220405, + "tokens_seen": 899940352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673319959879639, + "loss": 2.6941, + "theoretical_loss": 3.6863221435530567, + "tokens_seen": 900005888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673219658976931, + "loss": 3.0026, + "theoretical_loss": 3.6862960995116505, + "tokens_seen": 900071424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036731193580742227, + "loss": 2.6116, + "theoretical_loss": 3.6862700578974192, + "tokens_seen": 900136960 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036730190571715145, + "loss": 2.8755, + "theoretical_loss": 3.68624401870996, + "tokens_seen": 900202496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036729187562688063, + "loss": 2.4658, + "theoretical_loss": 3.68621798194887, + "tokens_seen": 900268032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672818455366098, + "loss": 2.8991, + "theoretical_loss": 3.686191947613746, + "tokens_seen": 900333568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036727181544633905, + "loss": 2.972, + "theoretical_loss": 3.686165915704186, + "tokens_seen": 900399104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672617853560682, + "loss": 2.9544, + "theoretical_loss": 3.686139886219788, + "tokens_seen": 900464640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672517552657974, + "loss": 2.7917, + "theoretical_loss": 3.6861138591601486, + "tokens_seen": 900530176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036724172517552654, + "loss": 2.7219, + "theoretical_loss": 3.6860878345248658, + "tokens_seen": 900595712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036723169508525577, + "loss": 2.7, + "theoretical_loss": 3.686061812313538, + "tokens_seen": 900661248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036722166499498495, + "loss": 2.9898, + "theoretical_loss": 3.686035792525762, + "tokens_seen": 900726784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036721163490471414, + "loss": 2.7812, + "theoretical_loss": 3.6860097751611356, + "tokens_seen": 900792320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672016048144433, + "loss": 2.7424, + "theoretical_loss": 3.6859837602192584, + "tokens_seen": 900857856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036719157472417255, + "loss": 2.9703, + "theoretical_loss": 3.6859577476997276, + "tokens_seen": 900923392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671815446339017, + "loss": 2.8666, + "theoretical_loss": 3.6859317376021417, + "tokens_seen": 900988928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671715145436309, + "loss": 2.7763, + "theoretical_loss": 3.6859057299260987, + "tokens_seen": 901054464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 504328, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.588376045227051, + "objective/train/theoretical_loss": 3.6858797246711976, + "objective/train/tokens_used": 921580000, + "theoretical_loss": 3.6858797246711976, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036716148445336004, + "loss": 2.7931, + "theoretical_loss": 3.6858797246711976, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671514543630893, + "loss": 2.7644, + "theoretical_loss": 3.685853721837036, + "tokens_seen": 901185536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036714142427281846, + "loss": 2.8663, + "theoretical_loss": 3.6858277214232142, + "tokens_seen": 901251072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036713139418254764, + "loss": 2.8726, + "theoretical_loss": 3.6858017234293294, + "tokens_seen": 901316608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671213640922768, + "loss": 2.8011, + "theoretical_loss": 3.6857757278549803, + "tokens_seen": 901382144 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367111334002006, + "loss": 2.9276, + "theoretical_loss": 3.6857497346997676, + "tokens_seen": 901447680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036710130391173524, + "loss": 2.8506, + "theoretical_loss": 3.6857237439632886, + "tokens_seen": 901513216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670912738214644, + "loss": 2.8113, + "theoretical_loss": 3.685697755645143, + "tokens_seen": 901578752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670812437311936, + "loss": 2.7378, + "theoretical_loss": 3.6856717697449306, + "tokens_seen": 901644288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670712136409228, + "loss": 2.7982, + "theoretical_loss": 3.68564578626225, + "tokens_seen": 901709824 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367061183550652, + "loss": 2.3642, + "theoretical_loss": 3.6856198051967004, + "tokens_seen": 901775360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036705115346038114, + "loss": 2.837, + "theoretical_loss": 3.6855938265478825, + "tokens_seen": 901840896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670411233701104, + "loss": 2.7009, + "theoretical_loss": 3.6855678503153944, + "tokens_seen": 901906432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670310932798395, + "loss": 2.8224, + "theoretical_loss": 3.685541876498837, + "tokens_seen": 901971968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036702106318956874, + "loss": 2.8333, + "theoretical_loss": 3.685515905097809, + "tokens_seen": 902037504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670110330992979, + "loss": 2.8069, + "theoretical_loss": 3.685489936111911, + "tokens_seen": 902103040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670010030090271, + "loss": 2.8903, + "theoretical_loss": 3.685463969540743, + "tokens_seen": 902168576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669909729187563, + "loss": 2.6957, + "theoretical_loss": 3.685438005383905, + "tokens_seen": 902234112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036698094282848546, + "loss": 2.7104, + "theoretical_loss": 3.685412043640997, + "tokens_seen": 902299648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036697091273821464, + "loss": 2.7687, + "theoretical_loss": 3.6853860843116193, + "tokens_seen": 902365184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669608826479439, + "loss": 2.8506, + "theoretical_loss": 3.6853601273953727, + "tokens_seen": 902430720 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366950852557673, + "loss": 2.6235, + "theoretical_loss": 3.6853341728918574, + "tokens_seen": 902496256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036694082246740224, + "loss": 2.8101, + "theoretical_loss": 3.6853082208006733, + "tokens_seen": 902561792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036693079237713137, + "loss": 2.9473, + "theoretical_loss": 3.685282271121422, + "tokens_seen": 902627328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669207622868606, + "loss": 2.7178, + "theoretical_loss": 3.6852563238537037, + "tokens_seen": 902692864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 504856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7679882049560547, + "objective/train/theoretical_loss": 3.6852303789971192, + "objective/train/tokens_used": 923218400, + "theoretical_loss": 3.6852303789971192, + "tokens_seen": 902758400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669107321965898, + "loss": 2.8659, + "theoretical_loss": 3.6852303789971192, + "tokens_seen": 902758400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036690070210631897, + "loss": 2.9552, + "theoretical_loss": 3.68520443655127, + "tokens_seen": 902823936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036689067201604815, + "loss": 2.63, + "theoretical_loss": 3.6851784965157566, + "tokens_seen": 902889472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668806419257774, + "loss": 2.665, + "theoretical_loss": 3.68515255889018, + "tokens_seen": 902955008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668706118355065, + "loss": 2.7384, + "theoretical_loss": 3.685126623674142, + "tokens_seen": 903020544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036686058174523575, + "loss": 2.8029, + "theoretical_loss": 3.6851006908672437, + "tokens_seen": 903086080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036685055165496487, + "loss": 2.7919, + "theoretical_loss": 3.685074760469086, + "tokens_seen": 903151616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668405215646941, + "loss": 2.74, + "theoretical_loss": 3.6850488324792714, + "tokens_seen": 903217152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668304914744233, + "loss": 2.8418, + "theoretical_loss": 3.6850229068974008, + "tokens_seen": 903282688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036682046138415247, + "loss": 2.7863, + "theoretical_loss": 3.6849969837230754, + "tokens_seen": 903348224 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036681043129388165, + "loss": 2.8803, + "theoretical_loss": 3.6849710629558983, + "tokens_seen": 903413760 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036680040120361083, + "loss": 2.9019, + "theoretical_loss": 3.6849451445954706, + "tokens_seen": 903479296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036679037111334, + "loss": 2.836, + "theoretical_loss": 3.684919228641394, + "tokens_seen": 903544832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036678034102306925, + "loss": 2.7521, + "theoretical_loss": 3.6848933150932712, + "tokens_seen": 903610368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667703109327984, + "loss": 2.5911, + "theoretical_loss": 3.684867403950704, + "tokens_seen": 903675904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667602808425276, + "loss": 2.6519, + "theoretical_loss": 3.684841495213295, + "tokens_seen": 903741440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036675025075225674, + "loss": 2.8198, + "theoretical_loss": 3.684815588880646, + "tokens_seen": 903806976 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366740220661986, + "loss": 2.5777, + "theoretical_loss": 3.68478968495236, + "tokens_seen": 903872512 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036673019057171515, + "loss": 2.875, + "theoretical_loss": 3.684763783428039, + "tokens_seen": 903938048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036672016048144434, + "loss": 2.7931, + "theoretical_loss": 3.684737884307286, + "tokens_seen": 904003584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667101303911735, + "loss": 2.6516, + "theoretical_loss": 3.6847119875897043, + "tokens_seen": 904069120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036670010030090275, + "loss": 2.8678, + "theoretical_loss": 3.684686093274896, + "tokens_seen": 904134656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666900702106319, + "loss": 2.6876, + "theoretical_loss": 3.6846602013624636, + "tokens_seen": 904200192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666800401203611, + "loss": 2.5903, + "theoretical_loss": 3.684634311852011, + "tokens_seen": 904265728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036667001003009024, + "loss": 2.8271, + "theoretical_loss": 3.684608424743141, + "tokens_seen": 904331264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 505908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.720822811126709, + "objective/train/theoretical_loss": 3.684582540035456, + "objective/train/tokens_used": 924856800, + "theoretical_loss": 3.684582540035456, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666599799398195, + "loss": 2.7747, + "theoretical_loss": 3.684582540035456, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036664994984954866, + "loss": 2.5759, + "theoretical_loss": 3.684556657728561, + "tokens_seen": 904462336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036663991975927784, + "loss": 2.8172, + "theoretical_loss": 3.684530777822059, + "tokens_seen": 904527872 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366629889669007, + "loss": 2.6142, + "theoretical_loss": 3.684504900315552, + "tokens_seen": 904593408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666198595787362, + "loss": 2.8247, + "theoretical_loss": 3.6844790252086446, + "tokens_seen": 904658944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666098294884654, + "loss": 2.7764, + "theoretical_loss": 3.6844531525009403, + "tokens_seen": 904724480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665997993981946, + "loss": 2.8979, + "theoretical_loss": 3.6844272821920434, + "tokens_seen": 904790016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036658976930792374, + "loss": 2.7468, + "theoretical_loss": 3.6844014142815578, + "tokens_seen": 904855552 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366579739217653, + "loss": 2.9656, + "theoretical_loss": 3.6843755487690864, + "tokens_seen": 904921088 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036656970912738216, + "loss": 2.6941, + "theoretical_loss": 3.684349685654234, + "tokens_seen": 904986624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036655967903711134, + "loss": 2.6184, + "theoretical_loss": 3.6843238249366044, + "tokens_seen": 905052160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665496489468405, + "loss": 2.659, + "theoretical_loss": 3.6842979666158024, + "tokens_seen": 905117696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665396188565697, + "loss": 2.9313, + "theoretical_loss": 3.6842721106914316, + "tokens_seen": 905183232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665295887662989, + "loss": 2.6039, + "theoretical_loss": 3.6842462571630974, + "tokens_seen": 905248768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665195586760281, + "loss": 3.0197, + "theoretical_loss": 3.6842204060304033, + "tokens_seen": 905314304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036650952858575725, + "loss": 2.7065, + "theoretical_loss": 3.6841945572929546, + "tokens_seen": 905379840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664994984954865, + "loss": 2.6915, + "theoretical_loss": 3.684168710950356, + "tokens_seen": 905445376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664894684052156, + "loss": 2.6842, + "theoretical_loss": 3.684142867002212, + "tokens_seen": 905510912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036647943831494484, + "loss": 2.6965, + "theoretical_loss": 3.684117025448127, + "tokens_seen": 905576448 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366469408224674, + "loss": 2.9644, + "theoretical_loss": 3.684091186287707, + "tokens_seen": 905641984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664593781344032, + "loss": 2.8037, + "theoretical_loss": 3.684065349520557, + "tokens_seen": 905707520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664493480441324, + "loss": 2.5095, + "theoretical_loss": 3.6840395151462815, + "tokens_seen": 905773056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036643931795386157, + "loss": 2.6812, + "theoretical_loss": 3.684013683164486, + "tokens_seen": 905838592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036642928786359075, + "loss": 2.852, + "theoretical_loss": 3.6839878535747763, + "tokens_seen": 905904128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036641925777332, + "loss": 2.6574, + "theoretical_loss": 3.683962026376758, + "tokens_seen": 905969664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 506404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0108041763305664, + "objective/train/theoretical_loss": 3.6839362015700354, + "objective/train/tokens_used": 926495200, + "theoretical_loss": 3.6839362015700354, + "tokens_seen": 906035200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664092276830491, + "loss": 2.5565, + "theoretical_loss": 3.6839362015700354, + "tokens_seen": 906035200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036639919759277835, + "loss": 2.7534, + "theoretical_loss": 3.683910379154215, + "tokens_seen": 906100736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036638916750250753, + "loss": 2.4499, + "theoretical_loss": 3.683884559128903, + "tokens_seen": 906166272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663791374122367, + "loss": 2.7014, + "theoretical_loss": 3.6838587414937045, + "tokens_seen": 906231808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663691073219659, + "loss": 2.6931, + "theoretical_loss": 3.6838329262482254, + "tokens_seen": 906297344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036635907723169507, + "loss": 2.9389, + "theoretical_loss": 3.6838071133920725, + "tokens_seen": 906362880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663490471414243, + "loss": 2.7749, + "theoretical_loss": 3.683781302924851, + "tokens_seen": 906428416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663390170511535, + "loss": 2.7101, + "theoretical_loss": 3.6837554948461677, + "tokens_seen": 906493952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036632898696088267, + "loss": 2.6792, + "theoretical_loss": 3.683729689155629, + "tokens_seen": 906559488 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036631895687061185, + "loss": 2.7912, + "theoretical_loss": 3.6837038858528404, + "tokens_seen": 906625024 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036630892678034103, + "loss": 2.728, + "theoretical_loss": 3.6836780849374096, + "tokens_seen": 906690560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662988966900702, + "loss": 2.6709, + "theoretical_loss": 3.6836522864089423, + "tokens_seen": 906756096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036628886659979945, + "loss": 2.8336, + "theoretical_loss": 3.683626490267046, + "tokens_seen": 906821632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662788365095286, + "loss": 2.7861, + "theoretical_loss": 3.683600696511326, + "tokens_seen": 906887168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662688064192578, + "loss": 2.8822, + "theoretical_loss": 3.6835749051413904, + "tokens_seen": 906952704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036625877632898694, + "loss": 2.8046, + "theoretical_loss": 3.683549116156846, + "tokens_seen": 907018240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662487462387162, + "loss": 2.6543, + "theoretical_loss": 3.6835233295573, + "tokens_seen": 907083776 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036623871614844535, + "loss": 2.9645, + "theoretical_loss": 3.6834975453423593, + "tokens_seen": 907149312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036622868605817454, + "loss": 2.7094, + "theoretical_loss": 3.6834717635116307, + "tokens_seen": 907214848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662186559679037, + "loss": 2.7114, + "theoretical_loss": 3.683445984064722, + "tokens_seen": 907280384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036620862587763295, + "loss": 2.6491, + "theoretical_loss": 3.6834202070012405, + "tokens_seen": 907345920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003661985957873621, + "loss": 2.8925, + "theoretical_loss": 3.6833944323207937, + "tokens_seen": 907411456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003661885656970913, + "loss": 2.9666, + "theoretical_loss": 3.68336866002299, + "tokens_seen": 907476992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036617853560682044, + "loss": 2.6134, + "theoretical_loss": 3.6833428901074354, + "tokens_seen": 907542528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661685055165497, + "loss": 2.7143, + "theoretical_loss": 3.683317122573739, + "tokens_seen": 907608064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 507582, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5680878162384033, + "objective/train/theoretical_loss": 3.683291357421508, + "objective/train/tokens_used": 928133600, + "theoretical_loss": 3.683291357421508, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036615847542627886, + "loss": 2.7327, + "theoretical_loss": 3.683291357421508, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036614844533600804, + "loss": 2.8497, + "theoretical_loss": 3.683265594650351, + "tokens_seen": 907739136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661384152457372, + "loss": 2.6495, + "theoretical_loss": 3.683239834259876, + "tokens_seen": 907804672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661283851554664, + "loss": 2.9766, + "theoretical_loss": 3.6832140762496905, + "tokens_seen": 907870208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661183550651956, + "loss": 2.7795, + "theoretical_loss": 3.683188320619404, + "tokens_seen": 907935744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661083249749248, + "loss": 2.6712, + "theoretical_loss": 3.6831625673686235, + "tokens_seen": 908001280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036609829488465394, + "loss": 2.7726, + "theoretical_loss": 3.6831368164969582, + "tokens_seen": 908066816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660882647943832, + "loss": 2.7, + "theoretical_loss": 3.683111068004016, + "tokens_seen": 908132352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036607823470411236, + "loss": 2.6533, + "theoretical_loss": 3.6830853218894064, + "tokens_seen": 908197888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036606820461384154, + "loss": 2.6621, + "theoretical_loss": 3.6830595781527373, + "tokens_seen": 908263424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660581745235707, + "loss": 2.8009, + "theoretical_loss": 3.6830338367936184, + "tokens_seen": 908328960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660481444332999, + "loss": 2.6356, + "theoretical_loss": 3.683008097811658, + "tokens_seen": 908394496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660381143430291, + "loss": 2.8399, + "theoretical_loss": 3.6829823612064647, + "tokens_seen": 908460032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660280842527583, + "loss": 2.8992, + "theoretical_loss": 3.6829566269776484, + "tokens_seen": 908525568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036601805416248745, + "loss": 2.7555, + "theoretical_loss": 3.6829308951248185, + "tokens_seen": 908591104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660080240722167, + "loss": 2.9406, + "theoretical_loss": 3.682905165647583, + "tokens_seen": 908656640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659979939819458, + "loss": 2.7128, + "theoretical_loss": 3.6828794385455526, + "tokens_seen": 908722176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036598796389167504, + "loss": 2.6451, + "theoretical_loss": 3.6828537138183357, + "tokens_seen": 908787712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659779338014042, + "loss": 2.8677, + "theoretical_loss": 3.682827991465543, + "tokens_seen": 908853248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659679037111334, + "loss": 2.7044, + "theoretical_loss": 3.682802271486783, + "tokens_seen": 908918784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659578736208626, + "loss": 2.7429, + "theoretical_loss": 3.6827765538816655, + "tokens_seen": 908984320 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036594784353059177, + "loss": 2.8765, + "theoretical_loss": 3.6827508386498016, + "tokens_seen": 909049856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036593781344032095, + "loss": 2.6742, + "theoretical_loss": 3.6827251257907996, + "tokens_seen": 909115392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659277833500502, + "loss": 2.651, + "theoretical_loss": 3.682699415304271, + "tokens_seen": 909180928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659177532597793, + "loss": 2.6706, + "theoretical_loss": 3.682673707189825, + "tokens_seen": 909246464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 508185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.757549524307251, + "objective/train/theoretical_loss": 3.6826480014470717, + "objective/train/tokens_used": 929772000, + "theoretical_loss": 3.6826480014470717, + "tokens_seen": 909312000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036590772316950855, + "loss": 2.6325, + "theoretical_loss": 3.6826480014470717, + "tokens_seen": 909312000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036589769307923773, + "loss": 2.5434, + "theoretical_loss": 3.6826222980756222, + "tokens_seen": 909377536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658876629889669, + "loss": 2.9387, + "theoretical_loss": 3.6825965970750856, + "tokens_seen": 909443072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658776328986961, + "loss": 2.8113, + "theoretical_loss": 3.682570898445074, + "tokens_seen": 909508608 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036586760280842527, + "loss": 2.8491, + "theoretical_loss": 3.6825452021851963, + "tokens_seen": 909574144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036585757271815445, + "loss": 2.6226, + "theoretical_loss": 3.682519508295065, + "tokens_seen": 909639680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658475426278837, + "loss": 2.4749, + "theoretical_loss": 3.6824938167742887, + "tokens_seen": 909705216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658375125376128, + "loss": 2.7062, + "theoretical_loss": 3.6824681276224798, + "tokens_seen": 909770752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036582748244734205, + "loss": 2.5519, + "theoretical_loss": 3.682442440839249, + "tokens_seen": 909836288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658174523570712, + "loss": 2.7983, + "theoretical_loss": 3.6824167564242067, + "tokens_seen": 909901824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658074222668004, + "loss": 2.7716, + "theoretical_loss": 3.682391074376965, + "tokens_seen": 909967360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657973921765296, + "loss": 2.7686, + "theoretical_loss": 3.6823653946971344, + "tokens_seen": 910032896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657873620862588, + "loss": 2.6568, + "theoretical_loss": 3.682339717384326, + "tokens_seen": 910098432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036577733199598796, + "loss": 2.7357, + "theoretical_loss": 3.6823140424381524, + "tokens_seen": 910163968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036576730190571714, + "loss": 2.7161, + "theoretical_loss": 3.682288369858223, + "tokens_seen": 910229504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657572718154463, + "loss": 2.7993, + "theoretical_loss": 3.6822626996441516, + "tokens_seen": 910295040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036574724172517555, + "loss": 2.9071, + "theoretical_loss": 3.6822370317955486, + "tokens_seen": 910360576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657372116349047, + "loss": 2.5976, + "theoretical_loss": 3.682211366312026, + "tokens_seen": 910426112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657271815446339, + "loss": 2.7127, + "theoretical_loss": 3.6821857031931957, + "tokens_seen": 910491648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657171514543631, + "loss": 2.7244, + "theoretical_loss": 3.68216004243867, + "tokens_seen": 910557184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657071213640923, + "loss": 2.768, + "theoretical_loss": 3.6821343840480605, + "tokens_seen": 910622720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036569709127382146, + "loss": 2.6111, + "theoretical_loss": 3.682108728020979, + "tokens_seen": 910688256 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036568706118355064, + "loss": 2.8181, + "theoretical_loss": 3.6820830743570383, + "tokens_seen": 910753792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656770310932798, + "loss": 2.7025, + "theoretical_loss": 3.6820574230558503, + "tokens_seen": 910819328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036566700100300906, + "loss": 2.7488, + "theoretical_loss": 3.682031774117028, + "tokens_seen": 910884864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 509502, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5731163024902344, + "objective/train/theoretical_loss": 3.682006127540184, + "objective/train/tokens_used": 931410400, + "theoretical_loss": 3.682006127540184, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656569709127382, + "loss": 2.7661, + "theoretical_loss": 3.682006127540184, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656469408224674, + "loss": 2.9085, + "theoretical_loss": 3.6819804833249297, + "tokens_seen": 911015936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036563691073219655, + "loss": 2.7918, + "theoretical_loss": 3.681954841470879, + "tokens_seen": 911081472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656268806419258, + "loss": 2.6731, + "theoretical_loss": 3.6819292019776437, + "tokens_seen": 911147008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036561685055165496, + "loss": 2.7863, + "theoretical_loss": 3.6819035648448373, + "tokens_seen": 911212544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036560682046138414, + "loss": 2.8253, + "theoretical_loss": 3.6818779300720728, + "tokens_seen": 911278080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655967903711134, + "loss": 2.7498, + "theoretical_loss": 3.6818522976589625, + "tokens_seen": 911343616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036558676028084256, + "loss": 2.6159, + "theoretical_loss": 3.6818266676051206, + "tokens_seen": 911409152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036557673019057174, + "loss": 2.7005, + "theoretical_loss": 3.6818010399101597, + "tokens_seen": 911474688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655667001003009, + "loss": 2.6145, + "theoretical_loss": 3.6817754145736927, + "tokens_seen": 911540224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655566700100301, + "loss": 2.7677, + "theoretical_loss": 3.681749791595334, + "tokens_seen": 911605760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655466399197593, + "loss": 2.9102, + "theoretical_loss": 3.6817241709746966, + "tokens_seen": 911671296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655366098294885, + "loss": 2.8622, + "theoretical_loss": 3.681698552711394, + "tokens_seen": 911736832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036552657973921765, + "loss": 2.7773, + "theoretical_loss": 3.6816729368050396, + "tokens_seen": 911802368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655165496489469, + "loss": 2.6455, + "theoretical_loss": 3.6816473232552482, + "tokens_seen": 911867904 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365506519558676, + "loss": 2.699, + "theoretical_loss": 3.6816217120616326, + "tokens_seen": 911933440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036549648946840525, + "loss": 2.8395, + "theoretical_loss": 3.681596103223807, + "tokens_seen": 911998976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654864593781344, + "loss": 2.7838, + "theoretical_loss": 3.6815704967413865, + "tokens_seen": 912064512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654764292878636, + "loss": 2.7302, + "theoretical_loss": 3.6815448926139833, + "tokens_seen": 912130048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654663991975928, + "loss": 2.8579, + "theoretical_loss": 3.6815192908412135, + "tokens_seen": 912195584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036545636910732197, + "loss": 2.7153, + "theoretical_loss": 3.6814936914226903, + "tokens_seen": 912261120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036544633901705115, + "loss": 2.8546, + "theoretical_loss": 3.6814680943580287, + "tokens_seen": 912326656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654363089267804, + "loss": 2.5904, + "theoretical_loss": 3.681442499646842, + "tokens_seen": 912392192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654262788365095, + "loss": 2.6852, + "theoretical_loss": 3.6814169072887464, + "tokens_seen": 912457728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036541624874623875, + "loss": 2.6125, + "theoretical_loss": 3.6813913172833557, + "tokens_seen": 912523264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 510138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.649203062057495, + "objective/train/theoretical_loss": 3.6813657296302846, + "objective/train/tokens_used": 933048800, + "theoretical_loss": 3.6813657296302846, + "tokens_seen": 912588800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036540621865596793, + "loss": 2.6317, + "theoretical_loss": 3.6813657296302846, + "tokens_seen": 912588800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653961885656971, + "loss": 2.6273, + "theoretical_loss": 3.6813401443291482, + "tokens_seen": 912654336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653861584754263, + "loss": 2.8201, + "theoretical_loss": 3.681314561379562, + "tokens_seen": 912719872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036537612838515547, + "loss": 2.8762, + "theoretical_loss": 3.68128898078114, + "tokens_seen": 912785408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036536609829488465, + "loss": 2.7939, + "theoretical_loss": 3.6812634025334985, + "tokens_seen": 912850944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653560682046139, + "loss": 2.7877, + "theoretical_loss": 3.6812378266362513, + "tokens_seen": 912916480 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365346038114343, + "loss": 2.7212, + "theoretical_loss": 3.6812122530890146, + "tokens_seen": 912982016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036533600802407225, + "loss": 2.683, + "theoretical_loss": 3.681186681891404, + "tokens_seen": 913047552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653259779338014, + "loss": 2.7768, + "theoretical_loss": 3.6811611130430344, + "tokens_seen": 913113088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653159478435306, + "loss": 2.6558, + "theoretical_loss": 3.681135546543522, + "tokens_seen": 913178624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653059177532598, + "loss": 2.6317, + "theoretical_loss": 3.6811099823924813, + "tokens_seen": 913244160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365295887662989, + "loss": 2.7281, + "theoretical_loss": 3.6810844205895297, + "tokens_seen": 913309696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036528585757271816, + "loss": 2.6293, + "theoretical_loss": 3.6810588611342823, + "tokens_seen": 913375232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036527582748244734, + "loss": 2.7732, + "theoretical_loss": 3.6810333040263545, + "tokens_seen": 913440768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652657973921765, + "loss": 2.7447, + "theoretical_loss": 3.6810077492653632, + "tokens_seen": 913506304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036525576730190575, + "loss": 2.8119, + "theoretical_loss": 3.680982196850924, + "tokens_seen": 913571840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652457372116349, + "loss": 2.6916, + "theoretical_loss": 3.6809566467826533, + "tokens_seen": 913637376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652357071213641, + "loss": 2.8739, + "theoretical_loss": 3.680931099060168, + "tokens_seen": 913702912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652256770310933, + "loss": 2.6952, + "theoretical_loss": 3.680905553683083, + "tokens_seen": 913768448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652156469408225, + "loss": 2.7815, + "theoretical_loss": 3.680880010651016, + "tokens_seen": 913833984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036520561685055166, + "loss": 2.8346, + "theoretical_loss": 3.6808544699635837, + "tokens_seen": 913899520 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036519558676028084, + "loss": 2.5171, + "theoretical_loss": 3.680828931620402, + "tokens_seen": 913965056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036518555667001, + "loss": 2.6756, + "theoretical_loss": 3.680803395621088, + "tokens_seen": 914030592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036517552657973926, + "loss": 2.5853, + "theoretical_loss": 3.680777861965259, + "tokens_seen": 914096128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651654964894684, + "loss": 2.5823, + "theoretical_loss": 3.680752330652531, + "tokens_seen": 914161664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 511255, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8368189334869385, + "objective/train/theoretical_loss": 3.680726801682522, + "objective/train/tokens_used": 934687200, + "theoretical_loss": 3.680726801682522, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651554663991976, + "loss": 2.7186, + "theoretical_loss": 3.680726801682522, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036514543630892675, + "loss": 2.6055, + "theoretical_loss": 3.6807012750548482, + "tokens_seen": 914292736 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365135406218656, + "loss": 2.6287, + "theoretical_loss": 3.6806757507691277, + "tokens_seen": 914358272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036512537612838516, + "loss": 2.552, + "theoretical_loss": 3.6806502288249767, + "tokens_seen": 914423808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036511534603811434, + "loss": 2.6986, + "theoretical_loss": 3.6806247092220143, + "tokens_seen": 914489344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651053159478435, + "loss": 2.8004, + "theoretical_loss": 3.6805991919598564, + "tokens_seen": 914554880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036509528585757276, + "loss": 2.5273, + "theoretical_loss": 3.680573677038121, + "tokens_seen": 914620416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650852557673019, + "loss": 2.7229, + "theoretical_loss": 3.680548164456426, + "tokens_seen": 914685952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650752256770311, + "loss": 2.4541, + "theoretical_loss": 3.680522654214389, + "tokens_seen": 914751488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036506519558676025, + "loss": 2.6284, + "theoretical_loss": 3.680497146311628, + "tokens_seen": 914817024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650551654964895, + "loss": 2.7057, + "theoretical_loss": 3.6804716407477605, + "tokens_seen": 914882560 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036504513540621867, + "loss": 2.8252, + "theoretical_loss": 3.6804461375224053, + "tokens_seen": 914948096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036503510531594785, + "loss": 2.7718, + "theoretical_loss": 3.68042063663518, + "tokens_seen": 915013632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036502507522567703, + "loss": 2.7386, + "theoretical_loss": 3.6803951380857027, + "tokens_seen": 915079168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650150451354062, + "loss": 2.4951, + "theoretical_loss": 3.6803696418735914, + "tokens_seen": 915144704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650050150451354, + "loss": 2.8558, + "theoretical_loss": 3.6803441479984653, + "tokens_seen": 915210240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649949849548646, + "loss": 2.743, + "theoretical_loss": 3.6803186564599426, + "tokens_seen": 915275776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649949849548646, + "loss": 2.5659, + "theoretical_loss": 3.6802931672576413, + "tokens_seen": 915341312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036498495486459375, + "loss": 2.7712, + "theoretical_loss": 3.680267680391181, + "tokens_seen": 915406848 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364974924774323, + "loss": 2.741, + "theoretical_loss": 3.6802421958601794, + "tokens_seen": 915472384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649648946840521, + "loss": 2.5163, + "theoretical_loss": 3.6802167136642563, + "tokens_seen": 915537920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036495486459378135, + "loss": 2.8221, + "theoretical_loss": 3.68019123380303, + "tokens_seen": 915603456 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036494483450351053, + "loss": 2.6905, + "theoretical_loss": 3.6801657562761196, + "tokens_seen": 915668992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649348044132397, + "loss": 2.8177, + "theoretical_loss": 3.6801402810831445, + "tokens_seen": 915734528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649247743229689, + "loss": 2.8277, + "theoretical_loss": 3.6801148082237236, + "tokens_seen": 915800064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 511864, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.596713066101074, + "objective/train/theoretical_loss": 3.680089337697476, + "objective/train/tokens_used": 936325600, + "theoretical_loss": 3.680089337697476, + "tokens_seen": 915865600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036491474423269813, + "loss": 2.5618, + "theoretical_loss": 3.680089337697476, + "tokens_seen": 915865600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036490471414242726, + "loss": 2.7874, + "theoretical_loss": 3.6800638695040213, + "tokens_seen": 915931136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648946840521565, + "loss": 2.8756, + "theoretical_loss": 3.680038403642979, + "tokens_seen": 915996672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648846539618856, + "loss": 2.9633, + "theoretical_loss": 3.6800129401139685, + "tokens_seen": 916062208 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036487462387161485, + "loss": 2.4687, + "theoretical_loss": 3.67998747891661, + "tokens_seen": 916127744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648645937813441, + "loss": 2.6901, + "theoretical_loss": 3.6799620200505223, + "tokens_seen": 916193280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648545636910732, + "loss": 2.4379, + "theoretical_loss": 3.6799365635153256, + "tokens_seen": 916258816 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036484453360080245, + "loss": 2.813, + "theoretical_loss": 3.6799111093106402, + "tokens_seen": 916324352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648345035105316, + "loss": 2.7068, + "theoretical_loss": 3.6798856574360856, + "tokens_seen": 916389888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648244734202608, + "loss": 2.9076, + "theoretical_loss": 3.6798602078912817, + "tokens_seen": 916455424 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036481444332999, + "loss": 2.7046, + "theoretical_loss": 3.679834760675849, + "tokens_seen": 916520960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648044132397192, + "loss": 2.6643, + "theoretical_loss": 3.679809315789408, + "tokens_seen": 916586496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036479438314944836, + "loss": 2.9033, + "theoretical_loss": 3.679783873231579, + "tokens_seen": 916652032 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036478435305917754, + "loss": 2.7993, + "theoretical_loss": 3.679758433001982, + "tokens_seen": 916717568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647743229689067, + "loss": 2.7358, + "theoretical_loss": 3.6797329951002373, + "tokens_seen": 916783104 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036476429287863595, + "loss": 2.7013, + "theoretical_loss": 3.6797075595259665, + "tokens_seen": 916848640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647542627883651, + "loss": 2.4434, + "theoretical_loss": 3.6796821262787898, + "tokens_seen": 916914176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647442326980943, + "loss": 2.5695, + "theoretical_loss": 3.679656695358328, + "tokens_seen": 916979712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647342026078235, + "loss": 2.8252, + "theoretical_loss": 3.679631266764201, + "tokens_seen": 917045248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647241725175527, + "loss": 2.8018, + "theoretical_loss": 3.6796058404960315, + "tokens_seen": 917110784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036471414242728186, + "loss": 2.804, + "theoretical_loss": 3.6795804165534394, + "tokens_seen": 917176320 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036470411233701104, + "loss": 2.869, + "theoretical_loss": 3.6795549949360464, + "tokens_seen": 917241856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646940822467402, + "loss": 2.7698, + "theoretical_loss": 3.6795295756434734, + "tokens_seen": 917307392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036468405215646946, + "loss": 2.5769, + "theoretical_loss": 3.679504158675342, + "tokens_seen": 917372928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646740220661986, + "loss": 2.6722, + "theoretical_loss": 3.6794787440312735, + "tokens_seen": 917438464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 513081, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8393032550811768, + "objective/train/theoretical_loss": 3.679453331710889, + "objective/train/tokens_used": 937964000, + "theoretical_loss": 3.679453331710889, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646639919759278, + "loss": 2.6943, + "theoretical_loss": 3.679453331710889, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036465396188565695, + "loss": 3.0041, + "theoretical_loss": 3.6794279217138106, + "tokens_seen": 917569536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646439317953862, + "loss": 2.7193, + "theoretical_loss": 3.6794025140396602, + "tokens_seen": 917635072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036463390170511536, + "loss": 2.8523, + "theoretical_loss": 3.6793771086880587, + "tokens_seen": 917700608 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036462387161484454, + "loss": 2.6422, + "theoretical_loss": 3.6793517056586285, + "tokens_seen": 917766144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646138415245737, + "loss": 2.8937, + "theoretical_loss": 3.6793263049509917, + "tokens_seen": 917831680 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036460381143430296, + "loss": 2.6182, + "theoretical_loss": 3.67930090656477, + "tokens_seen": 917897216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645937813440321, + "loss": 2.6326, + "theoretical_loss": 3.6792755104995853, + "tokens_seen": 917962752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645837512537613, + "loss": 2.6397, + "theoretical_loss": 3.6792501167550604, + "tokens_seen": 918028288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036457372116349045, + "loss": 2.5801, + "theoretical_loss": 3.679224725330818, + "tokens_seen": 918093824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645636910732197, + "loss": 2.7532, + "theoretical_loss": 3.679199336226479, + "tokens_seen": 918159360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036455366098294887, + "loss": 2.7627, + "theoretical_loss": 3.679173949441667, + "tokens_seen": 918224896 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036454363089267805, + "loss": 2.7622, + "theoretical_loss": 3.6791485649760043, + "tokens_seen": 918290432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036453360080240723, + "loss": 2.7045, + "theoretical_loss": 3.6791231828291133, + "tokens_seen": 918355968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645235707121364, + "loss": 2.7075, + "theoretical_loss": 3.679097803000617, + "tokens_seen": 918421504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645135406218656, + "loss": 2.8868, + "theoretical_loss": 3.6790724254901384, + "tokens_seen": 918487040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645035105315948, + "loss": 2.8046, + "theoretical_loss": 3.6790470502973, + "tokens_seen": 918552576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036449348044132395, + "loss": 2.6042, + "theoretical_loss": 3.679021677421725, + "tokens_seen": 918618112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644834503510532, + "loss": 2.8111, + "theoretical_loss": 3.678996306863037, + "tokens_seen": 918683648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644734202607823, + "loss": 2.713, + "theoretical_loss": 3.678970938620858, + "tokens_seen": 918749184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036446339017051155, + "loss": 2.6038, + "theoretical_loss": 3.678945572694812, + "tokens_seen": 918814720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036445336008024073, + "loss": 2.6464, + "theoretical_loss": 3.6789202090845223, + "tokens_seen": 918880256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644433299899699, + "loss": 2.7903, + "theoretical_loss": 3.6788948477896124, + "tokens_seen": 918945792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644332998996991, + "loss": 2.7264, + "theoretical_loss": 3.678869488809706, + "tokens_seen": 919011328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036442326980942833, + "loss": 2.5116, + "theoretical_loss": 3.6788441321444267, + "tokens_seen": 919076864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 514023, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6861374378204346, + "objective/train/theoretical_loss": 3.6788187777933974, + "objective/train/tokens_used": 939602400, + "theoretical_loss": 3.6788187777933974, + "tokens_seen": 919142400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036441323971915746, + "loss": 2.5727, + "theoretical_loss": 3.6788187777933974, + "tokens_seen": 919142400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644032096288867, + "loss": 2.7196, + "theoretical_loss": 3.678793425756243, + "tokens_seen": 919207936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643931795386158, + "loss": 2.6527, + "theoretical_loss": 3.6787680760325863, + "tokens_seen": 919273472 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036438314944834505, + "loss": 2.7031, + "theoretical_loss": 3.678742728622052, + "tokens_seen": 919339008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036437311935807424, + "loss": 2.7691, + "theoretical_loss": 3.6787173835242646, + "tokens_seen": 919404544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643630892678034, + "loss": 2.6177, + "theoretical_loss": 3.6786920407388473, + "tokens_seen": 919470080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643530591775326, + "loss": 2.7784, + "theoretical_loss": 3.678666700265425, + "tokens_seen": 919535616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643430290872618, + "loss": 2.586, + "theoretical_loss": 3.678641362103621, + "tokens_seen": 919601152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036433299899699096, + "loss": 2.5905, + "theoretical_loss": 3.678616026253061, + "tokens_seen": 919666688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643229689067202, + "loss": 2.6251, + "theoretical_loss": 3.6785906927133687, + "tokens_seen": 919732224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643129388164493, + "loss": 2.748, + "theoretical_loss": 3.678565361484169, + "tokens_seen": 919797760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036430290872617856, + "loss": 2.8521, + "theoretical_loss": 3.6785400325650865, + "tokens_seen": 919863296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642928786359077, + "loss": 2.71, + "theoretical_loss": 3.678514705955746, + "tokens_seen": 919928832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642828485456369, + "loss": 2.6943, + "theoretical_loss": 3.678489381655772, + "tokens_seen": 919994368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642728184553661, + "loss": 2.9214, + "theoretical_loss": 3.6784640596647904, + "tokens_seen": 920059904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642627883650953, + "loss": 2.8038, + "theoretical_loss": 3.6784387399824254, + "tokens_seen": 920125440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036425275827482446, + "loss": 2.9118, + "theoretical_loss": 3.678413422608302, + "tokens_seen": 920190976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642427281845537, + "loss": 2.4233, + "theoretical_loss": 3.6783881075420455, + "tokens_seen": 920256512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642326980942828, + "loss": 2.7865, + "theoretical_loss": 3.6783627947832818, + "tokens_seen": 920322048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036422266800401206, + "loss": 2.6493, + "theoretical_loss": 3.678337484331636, + "tokens_seen": 920387584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642126379137412, + "loss": 2.8862, + "theoretical_loss": 3.6783121761867332, + "tokens_seen": 920453120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642026078234704, + "loss": 2.6877, + "theoretical_loss": 3.678286870348199, + "tokens_seen": 920518656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641925777331996, + "loss": 2.7539, + "theoretical_loss": 3.6782615668156593, + "tokens_seen": 920584192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641825476429288, + "loss": 2.793, + "theoretical_loss": 3.67823626558874, + "tokens_seen": 920649728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036417251755265797, + "loss": 2.5625, + "theoretical_loss": 3.6782109666670664, + "tokens_seen": 920715264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 514680, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3389620780944824, + "objective/train/theoretical_loss": 3.6781856700502646, + "objective/train/tokens_used": 941240800, + "theoretical_loss": 3.6781856700502646, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036416248746238715, + "loss": 2.6741, + "theoretical_loss": 3.6781856700502646, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036415245737211633, + "loss": 2.5103, + "theoretical_loss": 3.678160375737961, + "tokens_seen": 920846336 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036414242728184556, + "loss": 2.7652, + "theoretical_loss": 3.678135083729781, + "tokens_seen": 920911872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641323971915747, + "loss": 2.6444, + "theoretical_loss": 3.6781097940253513, + "tokens_seen": 920977408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641223671013039, + "loss": 2.6945, + "theoretical_loss": 3.6780845066242978, + "tokens_seen": 921042944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036411233701103316, + "loss": 2.9332, + "theoretical_loss": 3.678059221526247, + "tokens_seen": 921108480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641023069207623, + "loss": 2.614, + "theoretical_loss": 3.678033938730825, + "tokens_seen": 921174016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640922768304915, + "loss": 2.7244, + "theoretical_loss": 3.6780086582376588, + "tokens_seen": 921239552 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036408224674022065, + "loss": 2.7352, + "theoretical_loss": 3.677983380046375, + "tokens_seen": 921305088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640722166499499, + "loss": 2.4935, + "theoretical_loss": 3.6779581041566, + "tokens_seen": 921370624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036406218655967907, + "loss": 2.4393, + "theoretical_loss": 3.6779328305679604, + "tokens_seen": 921436160 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036405215646940825, + "loss": 2.6316, + "theoretical_loss": 3.6779075592800834, + "tokens_seen": 921501696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036404212637913743, + "loss": 2.7714, + "theoretical_loss": 3.677882290292596, + "tokens_seen": 921567232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640320962888666, + "loss": 2.6272, + "theoretical_loss": 3.677857023605125, + "tokens_seen": 921632768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640220661985958, + "loss": 2.7397, + "theoretical_loss": 3.6778317592172973, + "tokens_seen": 921698304 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364012036108325, + "loss": 2.6956, + "theoretical_loss": 3.677806497128741, + "tokens_seen": 921763840 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036400200601805415, + "loss": 2.8998, + "theoretical_loss": 3.677781237339083, + "tokens_seen": 921829376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639919759277834, + "loss": 2.5165, + "theoretical_loss": 3.6777559798479498, + "tokens_seen": 921894912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639819458375125, + "loss": 2.685, + "theoretical_loss": 3.67773072465497, + "tokens_seen": 921960448 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036397191574724175, + "loss": 2.6519, + "theoretical_loss": 3.677705471759771, + "tokens_seen": 922025984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036396188565697093, + "loss": 2.6, + "theoretical_loss": 3.6776802211619795, + "tokens_seen": 922091520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639518555667001, + "loss": 2.7074, + "theoretical_loss": 3.677654972861224, + "tokens_seen": 922157056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639418254764293, + "loss": 2.7473, + "theoretical_loss": 3.6776297268571323, + "tokens_seen": 922222592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036393179538615853, + "loss": 2.8253, + "theoretical_loss": 3.6776044831493326, + "tokens_seen": 922288128 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036392176529588766, + "loss": 2.7029, + "theoretical_loss": 3.677579241737452, + "tokens_seen": 922353664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 515999, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7671639919281006, + "objective/train/theoretical_loss": 3.677554002621119, + "objective/train/tokens_used": 942879200, + "theoretical_loss": 3.677554002621119, + "tokens_seen": 922419200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639117352056169, + "loss": 2.6393, + "theoretical_loss": 3.677554002621119, + "tokens_seen": 922419200 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363901705115346, + "loss": 2.7322, + "theoretical_loss": 3.6775287657999627, + "tokens_seen": 922484736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036389167502507525, + "loss": 2.7465, + "theoretical_loss": 3.6775035312736097, + "tokens_seen": 922550272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036388164493480444, + "loss": 2.7354, + "theoretical_loss": 3.67747829904169, + "tokens_seen": 922615808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638716148445336, + "loss": 2.7393, + "theoretical_loss": 3.67745306910383, + "tokens_seen": 922681344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638615847542628, + "loss": 2.6674, + "theoretical_loss": 3.67742784145966, + "tokens_seen": 922746880 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363851554663992, + "loss": 2.9504, + "theoretical_loss": 3.6774026161088083, + "tokens_seen": 922812416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036384152457372116, + "loss": 2.645, + "theoretical_loss": 3.6773773930509024, + "tokens_seen": 922877952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638314944834504, + "loss": 2.6187, + "theoretical_loss": 3.6773521722855724, + "tokens_seen": 922943488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638214643931795, + "loss": 2.6137, + "theoretical_loss": 3.6773269538124467, + "tokens_seen": 923009024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036381143430290876, + "loss": 2.5773, + "theoretical_loss": 3.677301737631154, + "tokens_seen": 923074560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638014042126379, + "loss": 2.6507, + "theoretical_loss": 3.677276523741323, + "tokens_seen": 923140096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637913741223671, + "loss": 2.8147, + "theoretical_loss": 3.6772513121425847, + "tokens_seen": 923205632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637813440320963, + "loss": 2.9722, + "theoretical_loss": 3.677226102834566, + "tokens_seen": 923271168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637713139418255, + "loss": 2.8759, + "theoretical_loss": 3.677200895816897, + "tokens_seen": 923336704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036376128385155466, + "loss": 2.895, + "theoretical_loss": 3.6771756910892077, + "tokens_seen": 923402240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637512537612839, + "loss": 2.7822, + "theoretical_loss": 3.677150488651127, + "tokens_seen": 923467776 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363741223671013, + "loss": 2.752, + "theoretical_loss": 3.6771252885022845, + "tokens_seen": 923533312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036373119358074226, + "loss": 2.7301, + "theoretical_loss": 3.6771000906423095, + "tokens_seen": 923598848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637211634904714, + "loss": 2.8236, + "theoretical_loss": 3.6770748950708323, + "tokens_seen": 923664384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637111334002006, + "loss": 2.8326, + "theoretical_loss": 3.6770497017874826, + "tokens_seen": 923729920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637011033099298, + "loss": 2.6637, + "theoretical_loss": 3.67702451079189, + "tokens_seen": 923795456 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363691073219659, + "loss": 2.5562, + "theoretical_loss": 3.6769993220836845, + "tokens_seen": 923860992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036368104312938817, + "loss": 2.7717, + "theoretical_loss": 3.6769741356624968, + "tokens_seen": 923926528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036367101303911735, + "loss": 2.9733, + "theoretical_loss": 3.676948951527956, + "tokens_seen": 923992064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 516502, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.229719638824463, + "objective/train/theoretical_loss": 3.6769237696796933, + "objective/train/tokens_used": 944517600, + "theoretical_loss": 3.6769237696796933, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036366098294884653, + "loss": 2.9243, + "theoretical_loss": 3.6769237696796933, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036365095285857576, + "loss": 2.5783, + "theoretical_loss": 3.676898590117338, + "tokens_seen": 924123136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636409227683049, + "loss": 2.8002, + "theoretical_loss": 3.6768734128405214, + "tokens_seen": 924188672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636308926780341, + "loss": 2.5052, + "theoretical_loss": 3.6768482378488745, + "tokens_seen": 924254208 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036362086258776325, + "loss": 2.8707, + "theoretical_loss": 3.6768230651420257, + "tokens_seen": 924319744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636108324974925, + "loss": 2.6332, + "theoretical_loss": 3.6767978947196083, + "tokens_seen": 924385280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036360080240722167, + "loss": 2.8033, + "theoretical_loss": 3.676772726581251, + "tokens_seen": 924450816 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036359077231695085, + "loss": 2.6453, + "theoretical_loss": 3.676747560726586, + "tokens_seen": 924516352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036358074222668003, + "loss": 2.7782, + "theoretical_loss": 3.6767223971552427, + "tokens_seen": 924581888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036357071213640927, + "loss": 2.6504, + "theoretical_loss": 3.676697235866854, + "tokens_seen": 924647424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635606820461384, + "loss": 2.8365, + "theoretical_loss": 3.67667207686105, + "tokens_seen": 924712960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036355065195586763, + "loss": 2.769, + "theoretical_loss": 3.676646920137461, + "tokens_seen": 924778496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036354062186559676, + "loss": 2.5838, + "theoretical_loss": 3.67662176569572, + "tokens_seen": 924844032 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363530591775326, + "loss": 2.4492, + "theoretical_loss": 3.6765966135354575, + "tokens_seen": 924909568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036352056168505517, + "loss": 2.735, + "theoretical_loss": 3.676571463656305, + "tokens_seen": 924975104 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036351053159478435, + "loss": 2.6674, + "theoretical_loss": 3.6765463160578937, + "tokens_seen": 925040640 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036350050150451353, + "loss": 2.4657, + "theoretical_loss": 3.676521170739856, + "tokens_seen": 925106176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634904714142427, + "loss": 2.6863, + "theoretical_loss": 3.676496027701823, + "tokens_seen": 925171712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634804413239719, + "loss": 2.806, + "theoretical_loss": 3.6764708869434264, + "tokens_seen": 925237248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036347041123370113, + "loss": 2.6572, + "theoretical_loss": 3.676445748464298, + "tokens_seen": 925302784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036346038114343026, + "loss": 2.612, + "theoretical_loss": 3.67642061226407, + "tokens_seen": 925368320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634503510531595, + "loss": 2.6244, + "theoretical_loss": 3.676395478342375, + "tokens_seen": 925433856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634403209628886, + "loss": 2.7324, + "theoretical_loss": 3.676370346698844, + "tokens_seen": 925499392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036343029087261786, + "loss": 2.6495, + "theoretical_loss": 3.67634521733311, + "tokens_seen": 925564928 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036342026078234704, + "loss": 2.6261, + "theoretical_loss": 3.6763200902448054, + "tokens_seen": 925630464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 517571, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.828040361404419, + "objective/train/theoretical_loss": 3.676294965433562, + "objective/train/tokens_used": 946156000, + "theoretical_loss": 3.676294965433562, + "tokens_seen": 925696000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634102306920762, + "loss": 2.7145, + "theoretical_loss": 3.676294965433562, + "tokens_seen": 925696000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634002006018054, + "loss": 2.8367, + "theoretical_loss": 3.6762698428990124, + "tokens_seen": 925761536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036339017051153464, + "loss": 2.6012, + "theoretical_loss": 3.6762447226407895, + "tokens_seen": 925827072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036338014042126376, + "loss": 2.668, + "theoretical_loss": 3.6762196046585256, + "tokens_seen": 925892608 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363370110330993, + "loss": 2.8896, + "theoretical_loss": 3.676194488951853, + "tokens_seen": 925958144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633600802407222, + "loss": 2.8035, + "theoretical_loss": 3.6761693755204057, + "tokens_seen": 926023680 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036335005015045136, + "loss": 2.8345, + "theoretical_loss": 3.676144264363815, + "tokens_seen": 926089216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633400200601806, + "loss": 2.8302, + "theoretical_loss": 3.6761191554817154, + "tokens_seen": 926154752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633299899699097, + "loss": 2.5149, + "theoretical_loss": 3.6760940488737397, + "tokens_seen": 926220288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036331995987963896, + "loss": 2.4549, + "theoretical_loss": 3.6760689445395203, + "tokens_seen": 926285824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633099297893681, + "loss": 2.7747, + "theoretical_loss": 3.6760438424786903, + "tokens_seen": 926351360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632998996990973, + "loss": 2.7876, + "theoretical_loss": 3.676018742690884, + "tokens_seen": 926416896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632898696088265, + "loss": 2.7019, + "theoretical_loss": 3.6759936451757342, + "tokens_seen": 926482432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632798395185557, + "loss": 2.8405, + "theoretical_loss": 3.675968549932875, + "tokens_seen": 926547968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036326980942828486, + "loss": 2.6827, + "theoretical_loss": 3.6759434569619387, + "tokens_seen": 926613504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632597793380141, + "loss": 2.4907, + "theoretical_loss": 3.67591836626256, + "tokens_seen": 926679040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632497492477432, + "loss": 2.8732, + "theoretical_loss": 3.6758932778343727, + "tokens_seen": 926744576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036323971915747246, + "loss": 2.6314, + "theoretical_loss": 3.6758681916770097, + "tokens_seen": 926810112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632296890672016, + "loss": 2.7107, + "theoretical_loss": 3.675843107790106, + "tokens_seen": 926875648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632196589769308, + "loss": 2.7786, + "theoretical_loss": 3.675818026173295, + "tokens_seen": 926941184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036320962888666, + "loss": 2.6493, + "theoretical_loss": 3.6757929468262107, + "tokens_seen": 927006720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631995987963892, + "loss": 2.5168, + "theoretical_loss": 3.6757678697484875, + "tokens_seen": 927072256 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036318956870611837, + "loss": 2.7334, + "theoretical_loss": 3.67574279493976, + "tokens_seen": 927137792 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036317953861584755, + "loss": 2.6639, + "theoretical_loss": 3.6757177223996615, + "tokens_seen": 927203328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036316950852557673, + "loss": 2.7426, + "theoretical_loss": 3.675692652127827, + "tokens_seen": 927268864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 518862, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0217199325561523, + "objective/train/theoretical_loss": 3.6756675841238913, + "objective/train/tokens_used": 947794400, + "theoretical_loss": 3.6756675841238913, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036315947843530596, + "loss": 2.8993, + "theoretical_loss": 3.6756675841238913, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631494483450351, + "loss": 2.385, + "theoretical_loss": 3.6756425183874892, + "tokens_seen": 927399936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631394182547643, + "loss": 2.7425, + "theoretical_loss": 3.675617454918254, + "tokens_seen": 927465472 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036312938816449345, + "loss": 2.7258, + "theoretical_loss": 3.675592393715822, + "tokens_seen": 927531008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631193580742227, + "loss": 2.6855, + "theoretical_loss": 3.675567334779827, + "tokens_seen": 927596544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036310932798395187, + "loss": 3.0311, + "theoretical_loss": 3.675542278109905, + "tokens_seen": 927662080 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036309929789368105, + "loss": 2.6216, + "theoretical_loss": 3.6755172237056897, + "tokens_seen": 927727616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036308926780341023, + "loss": 2.2776, + "theoretical_loss": 3.675492171566817, + "tokens_seen": 927793152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036307923771313947, + "loss": 2.6105, + "theoretical_loss": 3.675467121692922, + "tokens_seen": 927858688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630692076228686, + "loss": 2.8852, + "theoretical_loss": 3.67544207408364, + "tokens_seen": 927924224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036305917753259783, + "loss": 2.6106, + "theoretical_loss": 3.675417028738606, + "tokens_seen": 927989760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036304914744232696, + "loss": 2.7, + "theoretical_loss": 3.675391985657456, + "tokens_seen": 928055296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630391173520562, + "loss": 2.6264, + "theoretical_loss": 3.675366944839825, + "tokens_seen": 928120832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036302908726178537, + "loss": 2.6397, + "theoretical_loss": 3.6753419062853485, + "tokens_seen": 928186368 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036301905717151455, + "loss": 2.5242, + "theoretical_loss": 3.6753168699936625, + "tokens_seen": 928251904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036300902708124374, + "loss": 2.6, + "theoretical_loss": 3.675291835964403, + "tokens_seen": 928317440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629989969909729, + "loss": 2.7545, + "theoretical_loss": 3.675266804197206, + "tokens_seen": 928382976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629889669007021, + "loss": 2.8854, + "theoretical_loss": 3.6752417746917065, + "tokens_seen": 928448512 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036297893681043133, + "loss": 2.6897, + "theoretical_loss": 3.675216747447541, + "tokens_seen": 928514048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036296890672016046, + "loss": 2.7338, + "theoretical_loss": 3.675191722464346, + "tokens_seen": 928579584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629588766298897, + "loss": 2.6159, + "theoretical_loss": 3.6751666997417574, + "tokens_seen": 928645120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629488465396188, + "loss": 2.8166, + "theoretical_loss": 3.6751416792794114, + "tokens_seen": 928710656 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036293881644934806, + "loss": 2.6078, + "theoretical_loss": 3.6751166610769443, + "tokens_seen": 928776192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036292878635907724, + "loss": 2.8892, + "theoretical_loss": 3.6750916451339926, + "tokens_seen": 928841728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629187562688064, + "loss": 2.4984, + "theoretical_loss": 3.6750666314501927, + "tokens_seen": 928907264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 519491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.917207717895508, + "objective/train/theoretical_loss": 3.675041620025182, + "objective/train/tokens_used": 949432800, + "theoretical_loss": 3.675041620025182, + "tokens_seen": 928972800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629087261785356, + "loss": 2.6886, + "theoretical_loss": 3.675041620025182, + "tokens_seen": 928972800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036289869608826484, + "loss": 2.6166, + "theoretical_loss": 3.675016610858596, + "tokens_seen": 929038336 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036288866599799396, + "loss": 2.7116, + "theoretical_loss": 3.6749916039500725, + "tokens_seen": 929103872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628786359077232, + "loss": 2.5936, + "theoretical_loss": 3.674966599299248, + "tokens_seen": 929169408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628686058174523, + "loss": 2.6567, + "theoretical_loss": 3.674941596905759, + "tokens_seen": 929234944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036285857572718156, + "loss": 2.7593, + "theoretical_loss": 3.6749165967692434, + "tokens_seen": 929300480 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036284854563691074, + "loss": 2.6485, + "theoretical_loss": 3.674891598889338, + "tokens_seen": 929366016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628385155466399, + "loss": 2.7085, + "theoretical_loss": 3.674866603265679, + "tokens_seen": 929431552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628284854563691, + "loss": 2.5491, + "theoretical_loss": 3.674841609897905, + "tokens_seen": 929497088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628184553660983, + "loss": 2.6976, + "theoretical_loss": 3.674816618785653, + "tokens_seen": 929562624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036280842527582747, + "loss": 2.5553, + "theoretical_loss": 3.67479162992856, + "tokens_seen": 929628160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627983951855567, + "loss": 2.8656, + "theoretical_loss": 3.6747666433262642, + "tokens_seen": 929693696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036278836509528583, + "loss": 2.6386, + "theoretical_loss": 3.674741658978403, + "tokens_seen": 929759232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036277833500501506, + "loss": 2.6119, + "theoretical_loss": 3.6747166768846142, + "tokens_seen": 929824768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036276830491474424, + "loss": 2.6403, + "theoretical_loss": 3.674691697044535, + "tokens_seen": 929890304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627582748244734, + "loss": 2.7474, + "theoretical_loss": 3.6746667194578038, + "tokens_seen": 929955840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627482447342026, + "loss": 2.6571, + "theoretical_loss": 3.6746417441240586, + "tokens_seen": 930021376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627382146439318, + "loss": 2.82, + "theoretical_loss": 3.674616771042937, + "tokens_seen": 930086912 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036272818455366097, + "loss": 2.8842, + "theoretical_loss": 3.6745918002140776, + "tokens_seen": 930152448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627181544633902, + "loss": 2.7982, + "theoretical_loss": 3.6745668316371183, + "tokens_seen": 930217984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036270812437311933, + "loss": 2.6886, + "theoretical_loss": 3.6745418653116975, + "tokens_seen": 930283520 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036269809428284857, + "loss": 2.7248, + "theoretical_loss": 3.6745169012374537, + "tokens_seen": 930349056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626880641925777, + "loss": 2.6175, + "theoretical_loss": 3.6744919394140254, + "tokens_seen": 930414592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036267803410230693, + "loss": 2.8798, + "theoretical_loss": 3.6744669798410508, + "tokens_seen": 930480128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626680040120361, + "loss": 2.7445, + "theoretical_loss": 3.6744420225181686, + "tokens_seen": 930545664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 520743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.064671277999878, + "objective/train/theoretical_loss": 3.6744170674450176, + "objective/train/tokens_used": 951071200, + "theoretical_loss": 3.6744170674450176, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626579739217653, + "loss": 2.841, + "theoretical_loss": 3.6744170674450176, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036264794383149447, + "loss": 2.5725, + "theoretical_loss": 3.6743921146212366, + "tokens_seen": 930676736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036263791374122365, + "loss": 2.7547, + "theoretical_loss": 3.6743671640464646, + "tokens_seen": 930742272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036262788365095283, + "loss": 2.7484, + "theoretical_loss": 3.6743422157203405, + "tokens_seen": 930807808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036261785356068207, + "loss": 2.5509, + "theoretical_loss": 3.6743172696425033, + "tokens_seen": 930873344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036260782347041125, + "loss": 2.5975, + "theoretical_loss": 3.6742923258125915, + "tokens_seen": 930938880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036259779338014043, + "loss": 2.8421, + "theoretical_loss": 3.674267384230246, + "tokens_seen": 931004416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036258776328986967, + "loss": 2.928, + "theoretical_loss": 3.6742424448951043, + "tokens_seen": 931069952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625777331995988, + "loss": 2.7928, + "theoretical_loss": 3.6742175078068064, + "tokens_seen": 931135488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036256770310932803, + "loss": 2.6882, + "theoretical_loss": 3.674192572964992, + "tokens_seen": 931201024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036255767301905716, + "loss": 2.8744, + "theoretical_loss": 3.6741676403693004, + "tokens_seen": 931266560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625476429287864, + "loss": 2.51, + "theoretical_loss": 3.6741427100193715, + "tokens_seen": 931332096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625376128385156, + "loss": 2.8063, + "theoretical_loss": 3.6741177819148447, + "tokens_seen": 931397632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036252758274824475, + "loss": 2.8343, + "theoretical_loss": 3.67409285605536, + "tokens_seen": 931463168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036251755265797394, + "loss": 2.771, + "theoretical_loss": 3.674067932440557, + "tokens_seen": 931528704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625075225677031, + "loss": 2.5716, + "theoretical_loss": 3.674043011070076, + "tokens_seen": 931594240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624974924774323, + "loss": 2.7609, + "theoretical_loss": 3.674018091943557, + "tokens_seen": 931659776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036248746238716153, + "loss": 2.7397, + "theoretical_loss": 3.67399317506064, + "tokens_seen": 931725312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036247743229689066, + "loss": 2.6042, + "theoretical_loss": 3.673968260420965, + "tokens_seen": 931790848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624674022066199, + "loss": 2.5878, + "theoretical_loss": 3.6739433480241726, + "tokens_seen": 931856384 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362457372116349, + "loss": 2.8429, + "theoretical_loss": 3.6739184378699026, + "tokens_seen": 931921920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036244734202607826, + "loss": 2.447, + "theoretical_loss": 3.6738935299577964, + "tokens_seen": 931987456 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036243731193580744, + "loss": 2.7397, + "theoretical_loss": 3.6738686242874943, + "tokens_seen": 932052992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624272818455366, + "loss": 2.5438, + "theoretical_loss": 3.6738437208586365, + "tokens_seen": 932118528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624172517552658, + "loss": 2.8547, + "theoretical_loss": 3.6738188196708634, + "tokens_seen": 932184064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 521434, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9683523178100586, + "objective/train/theoretical_loss": 3.673793920723817, + "objective/train/tokens_used": 952709600, + "theoretical_loss": 3.673793920723817, + "tokens_seen": 932249600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036240722166499504, + "loss": 2.8287, + "theoretical_loss": 3.673793920723817, + "tokens_seen": 932249600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036239719157472416, + "loss": 2.4303, + "theoretical_loss": 3.673769024017137, + "tokens_seen": 932315136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623871614844534, + "loss": 2.56, + "theoretical_loss": 3.6737441295504647, + "tokens_seen": 932380672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623771313941825, + "loss": 2.6744, + "theoretical_loss": 3.6737192373234415, + "tokens_seen": 932446208 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036236710130391176, + "loss": 2.8192, + "theoretical_loss": 3.6736943473357084, + "tokens_seen": 932511744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036235707121364094, + "loss": 2.8758, + "theoretical_loss": 3.673669459586906, + "tokens_seen": 932577280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623470411233701, + "loss": 2.6161, + "theoretical_loss": 3.6736445740766763, + "tokens_seen": 932642816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623370110330993, + "loss": 2.5468, + "theoretical_loss": 3.673619690804661, + "tokens_seen": 932708352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623269809428285, + "loss": 2.8673, + "theoretical_loss": 3.6735948097705, + "tokens_seen": 932773888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036231695085255767, + "loss": 2.694, + "theoretical_loss": 3.6735699309738363, + "tokens_seen": 932839424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623069207622869, + "loss": 2.5771, + "theoretical_loss": 3.6735450544143107, + "tokens_seen": 932904960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036229689067201603, + "loss": 2.638, + "theoretical_loss": 3.6735201800915656, + "tokens_seen": 932970496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036228686058174526, + "loss": 2.8063, + "theoretical_loss": 3.6734953080052417, + "tokens_seen": 933036032 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036227683049147444, + "loss": 2.7244, + "theoretical_loss": 3.673470438154982, + "tokens_seen": 933101568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622668004012036, + "loss": 2.8428, + "theoretical_loss": 3.6734455705404283, + "tokens_seen": 933167104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622567703109328, + "loss": 2.7789, + "theoretical_loss": 3.6734207051612215, + "tokens_seen": 933232640 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362246740220662, + "loss": 2.8115, + "theoretical_loss": 3.673395842017005, + "tokens_seen": 933298176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036223671013039117, + "loss": 2.5103, + "theoretical_loss": 3.6733709811074204, + "tokens_seen": 933363712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622266800401204, + "loss": 2.604, + "theoretical_loss": 3.6733461224321102, + "tokens_seen": 933429248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036221664994984953, + "loss": 2.9288, + "theoretical_loss": 3.673321265990717, + "tokens_seen": 933494784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036220661985957877, + "loss": 2.6005, + "theoretical_loss": 3.673296411782882, + "tokens_seen": 933560320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621965897693079, + "loss": 2.5925, + "theoretical_loss": 3.6732715598082493, + "tokens_seen": 933625856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036218655967903713, + "loss": 2.6181, + "theoretical_loss": 3.6732467100664605, + "tokens_seen": 933691392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621765295887663, + "loss": 2.6131, + "theoretical_loss": 3.673221862557159, + "tokens_seen": 933756928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621664994984955, + "loss": 2.5553, + "theoretical_loss": 3.6731970172799864, + "tokens_seen": 933822464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 522330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3445885181427, + "objective/train/theoretical_loss": 3.673172174234587, + "objective/train/tokens_used": 954348000, + "theoretical_loss": 3.673172174234587, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036215646940822467, + "loss": 2.617, + "theoretical_loss": 3.673172174234587, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036214643931795385, + "loss": 2.6809, + "theoretical_loss": 3.673147333420603, + "tokens_seen": 933953536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036213640922768303, + "loss": 2.634, + "theoretical_loss": 3.673122494837677, + "tokens_seen": 934019072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036212637913741227, + "loss": 2.6592, + "theoretical_loss": 3.6730976584854527, + "tokens_seen": 934084608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621163490471414, + "loss": 2.7614, + "theoretical_loss": 3.6730728243635733, + "tokens_seen": 934150144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036210631895687063, + "loss": 2.7846, + "theoretical_loss": 3.673047992471682, + "tokens_seen": 934215680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620962888665998, + "loss": 2.5949, + "theoretical_loss": 3.6730231628094216, + "tokens_seen": 934281216 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362086258776329, + "loss": 2.7675, + "theoretical_loss": 3.672998335376436, + "tokens_seen": 934346752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620762286860582, + "loss": 2.8264, + "theoretical_loss": 3.672973510172369, + "tokens_seen": 934412288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036206619859578736, + "loss": 2.6534, + "theoretical_loss": 3.6729486871968637, + "tokens_seen": 934477824 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036205616850551654, + "loss": 2.724, + "theoretical_loss": 3.672923866449564, + "tokens_seen": 934543360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620461384152458, + "loss": 2.7642, + "theoretical_loss": 3.6728990479301133, + "tokens_seen": 934608896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620361083249749, + "loss": 2.8175, + "theoretical_loss": 3.672874231638156, + "tokens_seen": 934674432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036202607823470414, + "loss": 2.6981, + "theoretical_loss": 3.672849417573336, + "tokens_seen": 934739968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036201604814443326, + "loss": 2.4681, + "theoretical_loss": 3.672824605735297, + "tokens_seen": 934805504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620060180541625, + "loss": 2.7621, + "theoretical_loss": 3.6727997961236825, + "tokens_seen": 934871040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619959879638917, + "loss": 2.7309, + "theoretical_loss": 3.6727749887381376, + "tokens_seen": 934936576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036198595787362086, + "loss": 2.7253, + "theoretical_loss": 3.672750183578306, + "tokens_seen": 935002112 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036197592778335004, + "loss": 2.8102, + "theoretical_loss": 3.6727253806438327, + "tokens_seen": 935067648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619658976930792, + "loss": 2.5928, + "theoretical_loss": 3.6727005799343613, + "tokens_seen": 935133184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619558676028084, + "loss": 2.6811, + "theoretical_loss": 3.672675781449537, + "tokens_seen": 935198720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036194583751253764, + "loss": 2.7066, + "theoretical_loss": 3.6726509851890032, + "tokens_seen": 935264256 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036193580742226677, + "loss": 2.7786, + "theoretical_loss": 3.672626191152406, + "tokens_seen": 935329792 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361925777331996, + "loss": 2.6312, + "theoretical_loss": 3.67260139933939, + "tokens_seen": 935395328 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619157472417252, + "loss": 2.7573, + "theoretical_loss": 3.6725766097495987, + "tokens_seen": 935460864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 523607, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.482036590576172, + "objective/train/theoretical_loss": 3.672551822382678, + "objective/train/tokens_used": 955986400, + "theoretical_loss": 3.672551822382678, + "tokens_seen": 935526400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036190571715145436, + "loss": 2.4688, + "theoretical_loss": 3.672551822382678, + "tokens_seen": 935526400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036189568706118354, + "loss": 2.8175, + "theoretical_loss": 3.672527037238273, + "tokens_seen": 935591936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618856569709127, + "loss": 2.8405, + "theoretical_loss": 3.6725022543160284, + "tokens_seen": 935657472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618756268806419, + "loss": 2.688, + "theoretical_loss": 3.672477473615589, + "tokens_seen": 935723008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036186559679037114, + "loss": 2.5455, + "theoretical_loss": 3.6724526951366006, + "tokens_seen": 935788544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618555667001003, + "loss": 2.9469, + "theoretical_loss": 3.672427918878708, + "tokens_seen": 935854080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618455366098295, + "loss": 2.7401, + "theoretical_loss": 3.6724031448415575, + "tokens_seen": 935919616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618355065195587, + "loss": 2.937, + "theoretical_loss": 3.672378373024794, + "tokens_seen": 935985152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036182547642928787, + "loss": 2.4816, + "theoretical_loss": 3.672353603428063, + "tokens_seen": 936050688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618154463390171, + "loss": 2.674, + "theoretical_loss": 3.6723288360510096, + "tokens_seen": 936116224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036180541624874623, + "loss": 2.4672, + "theoretical_loss": 3.6723040708932806, + "tokens_seen": 936181760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036179538615847546, + "loss": 2.6237, + "theoretical_loss": 3.672279307954521, + "tokens_seen": 936247296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036178535606820464, + "loss": 2.7813, + "theoretical_loss": 3.6722545472343775, + "tokens_seen": 936312832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617753259779338, + "loss": 2.5787, + "theoretical_loss": 3.672229788732495, + "tokens_seen": 936378368 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361765295887663, + "loss": 2.676, + "theoretical_loss": 3.67220503244852, + "tokens_seen": 936443904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617552657973922, + "loss": 2.5799, + "theoretical_loss": 3.6721802783820987, + "tokens_seen": 936509440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036174523570712137, + "loss": 2.7577, + "theoretical_loss": 3.6721555265328774, + "tokens_seen": 936574976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617352056168506, + "loss": 2.7343, + "theoretical_loss": 3.6721307769005023, + "tokens_seen": 936640512 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036172517552657973, + "loss": 2.6231, + "theoretical_loss": 3.6721060294846195, + "tokens_seen": 936706048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036171514543630897, + "loss": 2.4971, + "theoretical_loss": 3.6720812842848756, + "tokens_seen": 936771584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617051153460381, + "loss": 2.7271, + "theoretical_loss": 3.6720565413009174, + "tokens_seen": 936837120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036169508525576733, + "loss": 2.8992, + "theoretical_loss": 3.672031800532391, + "tokens_seen": 936902656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616850551654965, + "loss": 2.5012, + "theoretical_loss": 3.6720070619789436, + "tokens_seen": 936968192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616750250752257, + "loss": 2.7448, + "theoretical_loss": 3.671982325640222, + "tokens_seen": 937033728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036166499498495487, + "loss": 2.5139, + "theoretical_loss": 3.6719575915158726, + "tokens_seen": 937099264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 524349, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2419161796569824, + "objective/train/theoretical_loss": 3.6719328596055423, + "objective/train/tokens_used": 957624800, + "theoretical_loss": 3.6719328596055423, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036165496489468405, + "loss": 2.5676, + "theoretical_loss": 3.6719328596055423, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036164493480441323, + "loss": 2.8033, + "theoretical_loss": 3.6719081299088785, + "tokens_seen": 937230336 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036163490471414247, + "loss": 2.5401, + "theoretical_loss": 3.671883402425528, + "tokens_seen": 937295872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616248746238716, + "loss": 2.4762, + "theoretical_loss": 3.671858677155138, + "tokens_seen": 937361408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036161484453360083, + "loss": 2.4731, + "theoretical_loss": 3.671833954097356, + "tokens_seen": 937426944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036160481444333, + "loss": 2.777, + "theoretical_loss": 3.6718092332518295, + "tokens_seen": 937492480 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036160481444333, + "loss": 2.7376, + "theoretical_loss": 3.671784514618205, + "tokens_seen": 937558016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615947843530592, + "loss": 2.9631, + "theoretical_loss": 3.671759798196131, + "tokens_seen": 937623552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615847542627884, + "loss": 2.8124, + "theoretical_loss": 3.6717350839852547, + "tokens_seen": 937689088 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036157472417251756, + "loss": 2.8745, + "theoretical_loss": 3.671710371985223, + "tokens_seen": 937754624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036156469408224674, + "loss": 2.65, + "theoretical_loss": 3.6716856621956855, + "tokens_seen": 937820160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361554663991976, + "loss": 2.861, + "theoretical_loss": 3.6716609546162884, + "tokens_seen": 937885696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615446339017051, + "loss": 2.6636, + "theoretical_loss": 3.6716362492466796, + "tokens_seen": 937951232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036153460381143434, + "loss": 2.8794, + "theoretical_loss": 3.6716115460865084, + "tokens_seen": 938016768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036152457372116346, + "loss": 2.7265, + "theoretical_loss": 3.6715868451354217, + "tokens_seen": 938082304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615145436308927, + "loss": 2.5481, + "theoretical_loss": 3.671562146393068, + "tokens_seen": 938147840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615045135406219, + "loss": 2.7664, + "theoretical_loss": 3.671537449859095, + "tokens_seen": 938213376 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036149448345035106, + "loss": 2.7306, + "theoretical_loss": 3.6715127555331524, + "tokens_seen": 938278912 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036148445336008024, + "loss": 2.5884, + "theoretical_loss": 3.671488063414887, + "tokens_seen": 938344448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614744232698094, + "loss": 2.6665, + "theoretical_loss": 3.671463373503948, + "tokens_seen": 938409984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614643931795386, + "loss": 2.906, + "theoretical_loss": 3.671438685799984, + "tokens_seen": 938475520 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036145436308926784, + "loss": 2.8406, + "theoretical_loss": 3.671414000302644, + "tokens_seen": 938541056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036144433299899697, + "loss": 2.7746, + "theoretical_loss": 3.6713893170115757, + "tokens_seen": 938606592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614343029087262, + "loss": 2.8227, + "theoretical_loss": 3.6713646359264285, + "tokens_seen": 938672128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614242728184554, + "loss": 2.6755, + "theoretical_loss": 3.6713399570468512, + "tokens_seen": 938737664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 525468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.555947780609131, + "objective/train/theoretical_loss": 3.671315280372493, + "objective/train/tokens_used": 959263200, + "theoretical_loss": 3.671315280372493, + "tokens_seen": 938803200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036141424272818456, + "loss": 2.602, + "theoretical_loss": 3.671315280372493, + "tokens_seen": 938803200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036140421263791374, + "loss": 2.8354, + "theoretical_loss": 3.6712906059030024, + "tokens_seen": 938868736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613941825476429, + "loss": 2.774, + "theoretical_loss": 3.6712659336380287, + "tokens_seen": 938934272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613841524573721, + "loss": 2.714, + "theoretical_loss": 3.671241263577221, + "tokens_seen": 938999808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036137412236710134, + "loss": 2.7993, + "theoretical_loss": 3.671216595720229, + "tokens_seen": 939065344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036136409227683047, + "loss": 2.8738, + "theoretical_loss": 3.671191930066702, + "tokens_seen": 939130880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613540621865597, + "loss": 2.5938, + "theoretical_loss": 3.6711672666162887, + "tokens_seen": 939196416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036134403209628883, + "loss": 2.7551, + "theoretical_loss": 3.6711426053686393, + "tokens_seen": 939261952 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036133400200601807, + "loss": 2.9425, + "theoretical_loss": 3.6711179463234034, + "tokens_seen": 939327488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036132397191574725, + "loss": 2.5775, + "theoretical_loss": 3.6710932894802304, + "tokens_seen": 939393024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036131394182547643, + "loss": 2.737, + "theoretical_loss": 3.67106863483877, + "tokens_seen": 939458560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613039117352056, + "loss": 2.7386, + "theoretical_loss": 3.6710439823986722, + "tokens_seen": 939524096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036129388164493484, + "loss": 2.7826, + "theoretical_loss": 3.6710193321595868, + "tokens_seen": 939589632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036128385155466397, + "loss": 2.5974, + "theoretical_loss": 3.670994684121164, + "tokens_seen": 939655168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612738214643932, + "loss": 2.8526, + "theoretical_loss": 3.6709700382830537, + "tokens_seen": 939720704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036126379137412233, + "loss": 2.3337, + "theoretical_loss": 3.6709453946449058, + "tokens_seen": 939786240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036125376128385157, + "loss": 2.8718, + "theoretical_loss": 3.6709207532063712, + "tokens_seen": 939851776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036124373119358075, + "loss": 2.7124, + "theoretical_loss": 3.6708961139670997, + "tokens_seen": 939917312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036123370110330993, + "loss": 2.8604, + "theoretical_loss": 3.6708714769267417, + "tokens_seen": 939982848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612236710130391, + "loss": 2.9014, + "theoretical_loss": 3.6708468420849476, + "tokens_seen": 940048384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612136409227683, + "loss": 2.5795, + "theoretical_loss": 3.6708222094413685, + "tokens_seen": 940113920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612036108324975, + "loss": 2.7011, + "theoretical_loss": 3.670797578995655, + "tokens_seen": 940179456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611935807422267, + "loss": 2.8098, + "theoretical_loss": 3.6707729507474567, + "tokens_seen": 940244992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036118355065195584, + "loss": 2.5256, + "theoretical_loss": 3.670748324696426, + "tokens_seen": 940310528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036117352056168507, + "loss": 2.6763, + "theoretical_loss": 3.6707237008422116, + "tokens_seen": 940376064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 526013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.817310333251953, + "objective/train/theoretical_loss": 3.670699079184467, + "objective/train/tokens_used": 960901600, + "theoretical_loss": 3.670699079184467, + "tokens_seen": 940441600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611634904714142, + "loss": 2.6014, + "theoretical_loss": 3.670699079184467, + "tokens_seen": 940441600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036115346038114343, + "loss": 2.648, + "theoretical_loss": 3.6706744597228416, + "tokens_seen": 940507136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611434302908726, + "loss": 2.6984, + "theoretical_loss": 3.670649842456987, + "tokens_seen": 940572672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611334002006018, + "loss": 2.5761, + "theoretical_loss": 3.6706252273865543, + "tokens_seen": 940638208 + }, + { + "epoch": 0.29, + "learning_rate": 0.000361123370110331, + "loss": 2.6609, + "theoretical_loss": 3.6706006145111942, + "tokens_seen": 940703744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611133400200602, + "loss": 2.7398, + "theoretical_loss": 3.6705760038305595, + "tokens_seen": 940769280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611033099297894, + "loss": 2.6214, + "theoretical_loss": 3.6705513953443, + "tokens_seen": 940834816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610932798395186, + "loss": 2.7006, + "theoretical_loss": 3.6705267890520688, + "tokens_seen": 940900352 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036108324974924776, + "loss": 2.687, + "theoretical_loss": 3.670502184953516, + "tokens_seen": 940965888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036107321965897694, + "loss": 2.4898, + "theoretical_loss": 3.670477583048294, + "tokens_seen": 941031424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610631895687062, + "loss": 2.7117, + "theoretical_loss": 3.670452983336055, + "tokens_seen": 941096960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610531594784353, + "loss": 2.6429, + "theoretical_loss": 3.6704283858164497, + "tokens_seen": 941162496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036104312938816454, + "loss": 2.6568, + "theoretical_loss": 3.6704037904891305, + "tokens_seen": 941228032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036103309929789366, + "loss": 2.7248, + "theoretical_loss": 3.67037919735375, + "tokens_seen": 941293568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610230692076229, + "loss": 2.8445, + "theoretical_loss": 3.6703546064099593, + "tokens_seen": 941359104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610130391173521, + "loss": 2.8391, + "theoretical_loss": 3.6703300176574114, + "tokens_seen": 941424640 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036100300902708126, + "loss": 2.8193, + "theoretical_loss": 3.6703054310957577, + "tokens_seen": 941490176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036099297893681044, + "loss": 2.5903, + "theoretical_loss": 3.6702808467246513, + "tokens_seen": 941555712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609829488465396, + "loss": 2.6982, + "theoretical_loss": 3.670256264543744, + "tokens_seen": 941621248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609729187562688, + "loss": 2.7354, + "theoretical_loss": 3.6702316845526886, + "tokens_seen": 941686784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036096288866599804, + "loss": 2.869, + "theoretical_loss": 3.6702071067511373, + "tokens_seen": 941752320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036095285857572717, + "loss": 2.8926, + "theoretical_loss": 3.670182531138743, + "tokens_seen": 941817856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609428284854564, + "loss": 2.8551, + "theoretical_loss": 3.6701579577151584, + "tokens_seen": 941883392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609327983951856, + "loss": 2.7154, + "theoretical_loss": 3.670133386480036, + "tokens_seen": 941948928 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036092276830491476, + "loss": 2.5979, + "theoretical_loss": 3.6701088174330287, + "tokens_seen": 942014464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 527234, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.644130229949951, + "objective/train/theoretical_loss": 3.6700842505737903, + "objective/train/tokens_used": 962540000, + "theoretical_loss": 3.6700842505737903, + "tokens_seen": 942080000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036091273821464394, + "loss": 2.6034, + "theoretical_loss": 3.6700842505737903, + "tokens_seen": 942080000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609027081243731, + "loss": 2.6743, + "theoretical_loss": 3.6700596859019723, + "tokens_seen": 942145536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608926780341023, + "loss": 2.5416, + "theoretical_loss": 3.670035123417229, + "tokens_seen": 942211072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036088264794383154, + "loss": 2.5893, + "theoretical_loss": 3.670010563119213, + "tokens_seen": 942276608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036087261785356067, + "loss": 2.5442, + "theoretical_loss": 3.669986005007578, + "tokens_seen": 942342144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608625877632899, + "loss": 2.6865, + "theoretical_loss": 3.669961449081977, + "tokens_seen": 942407680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036085255767301903, + "loss": 2.372, + "theoretical_loss": 3.6699368953420635, + "tokens_seen": 942473216 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036084252758274827, + "loss": 2.467, + "theoretical_loss": 3.669912343787491, + "tokens_seen": 942538752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036083249749247745, + "loss": 2.874, + "theoretical_loss": 3.669887794417913, + "tokens_seen": 942604288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036082246740220663, + "loss": 2.83, + "theoretical_loss": 3.669863247232983, + "tokens_seen": 942669824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608124373119358, + "loss": 2.7182, + "theoretical_loss": 3.669838702232355, + "tokens_seen": 942735360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036080240722166505, + "loss": 2.5017, + "theoretical_loss": 3.669814159415683, + "tokens_seen": 942800896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036079237713139417, + "loss": 2.6341, + "theoretical_loss": 3.66978961878262, + "tokens_seen": 942866432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607823470411234, + "loss": 2.7817, + "theoretical_loss": 3.6697650803328212, + "tokens_seen": 942931968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036077231695085253, + "loss": 2.6504, + "theoretical_loss": 3.66974054406594, + "tokens_seen": 942997504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036076228686058177, + "loss": 2.7501, + "theoretical_loss": 3.66971600998163, + "tokens_seen": 943063040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036075225677031095, + "loss": 2.8004, + "theoretical_loss": 3.669691478079546, + "tokens_seen": 943128576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036074222668004013, + "loss": 2.7057, + "theoretical_loss": 3.6696669483593425, + "tokens_seen": 943194112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607321965897693, + "loss": 2.4509, + "theoretical_loss": 3.669642420820674, + "tokens_seen": 943259648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607221664994985, + "loss": 2.7561, + "theoretical_loss": 3.669617895463194, + "tokens_seen": 943325184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607121364092277, + "loss": 2.3886, + "theoretical_loss": 3.6695933722865575, + "tokens_seen": 943390720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607021063189569, + "loss": 2.7888, + "theoretical_loss": 3.6695688512904194, + "tokens_seen": 943456256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036069207622868604, + "loss": 2.7435, + "theoretical_loss": 3.669544332474434, + "tokens_seen": 943521792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036068204613841527, + "loss": 2.5464, + "theoretical_loss": 3.669519815838256, + "tokens_seen": 943587328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606720160481444, + "loss": 2.7557, + "theoretical_loss": 3.6694953013815406, + "tokens_seen": 943652864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 528141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.465517044067383, + "objective/train/theoretical_loss": 3.669470789103942, + "objective/train/tokens_used": 964178400, + "theoretical_loss": 3.669470789103942, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036066198595787364, + "loss": 2.7745, + "theoretical_loss": 3.669470789103942, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606519558676028, + "loss": 2.8102, + "theoretical_loss": 3.669446279005116, + "tokens_seen": 943783936 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360641925777332, + "loss": 2.6492, + "theoretical_loss": 3.6694217710847177, + "tokens_seen": 943849472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606318956870612, + "loss": 2.8314, + "theoretical_loss": 3.6693972653424014, + "tokens_seen": 943915008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606218655967904, + "loss": 2.6344, + "theoretical_loss": 3.669372761777823, + "tokens_seen": 943980544 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036061183550651954, + "loss": 2.7509, + "theoretical_loss": 3.669348260390638, + "tokens_seen": 944046080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606018054162488, + "loss": 3.0374, + "theoretical_loss": 3.669323761180501, + "tokens_seen": 944111616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605917753259779, + "loss": 2.965, + "theoretical_loss": 3.6692992641470683, + "tokens_seen": 944177152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036058174523570714, + "loss": 2.8973, + "theoretical_loss": 3.6692747692899945, + "tokens_seen": 944242688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605717151454363, + "loss": 2.8169, + "theoretical_loss": 3.6692502766089357, + "tokens_seen": 944308224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605616850551655, + "loss": 2.7218, + "theoretical_loss": 3.669225786103548, + "tokens_seen": 944373760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605516549648947, + "loss": 2.5733, + "theoretical_loss": 3.6692012977734865, + "tokens_seen": 944439296 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036054162487462386, + "loss": 2.6159, + "theoretical_loss": 3.6691768116184074, + "tokens_seen": 944504832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036053159478435304, + "loss": 2.5814, + "theoretical_loss": 3.669152327637967, + "tokens_seen": 944570368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605215646940823, + "loss": 2.8545, + "theoretical_loss": 3.66912784583182, + "tokens_seen": 944635904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605115346038114, + "loss": 2.841, + "theoretical_loss": 3.669103366199624, + "tokens_seen": 944701440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036050150451354064, + "loss": 2.5827, + "theoretical_loss": 3.6690788887410344, + "tokens_seen": 944766976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036049147442326977, + "loss": 2.6336, + "theoretical_loss": 3.6690544134557075, + "tokens_seen": 944832512 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360481444332999, + "loss": 2.6132, + "theoretical_loss": 3.6690299403432998, + "tokens_seen": 944898048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604714142427282, + "loss": 2.5371, + "theoretical_loss": 3.6690054694034675, + "tokens_seen": 944963584 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036046138415245737, + "loss": 2.5715, + "theoretical_loss": 3.6689810006358674, + "tokens_seen": 945029120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036045135406218655, + "loss": 2.7622, + "theoretical_loss": 3.6689565340401558, + "tokens_seen": 945094656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604413239719158, + "loss": 2.835, + "theoretical_loss": 3.6689320696159893, + "tokens_seen": 945160192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604312938816449, + "loss": 2.6585, + "theoretical_loss": 3.6689076073630242, + "tokens_seen": 945225728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036042126379137414, + "loss": 2.9013, + "theoretical_loss": 3.6688831472809182, + "tokens_seen": 945291264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 528757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7434470653533936, + "objective/train/theoretical_loss": 3.668858689369327, + "objective/train/tokens_used": 965816800, + "theoretical_loss": 3.668858689369327, + "tokens_seen": 945356800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036041123370110327, + "loss": 2.6086, + "theoretical_loss": 3.668858689369327, + "tokens_seen": 945356800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604012036108325, + "loss": 2.698, + "theoretical_loss": 3.6688342336279094, + "tokens_seen": 945422336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603911735205617, + "loss": 2.7111, + "theoretical_loss": 3.6688097800563213, + "tokens_seen": 945487872 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036038114343029087, + "loss": 2.7297, + "theoretical_loss": 3.668785328654219, + "tokens_seen": 945553408 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036037111334002005, + "loss": 2.8327, + "theoretical_loss": 3.668760879421261, + "tokens_seen": 945618944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036036108324974923, + "loss": 2.6444, + "theoretical_loss": 3.668736432357104, + "tokens_seen": 945684480 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036035105315947847, + "loss": 2.7526, + "theoretical_loss": 3.6687119874614047, + "tokens_seen": 945750016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036034102306920765, + "loss": 2.7224, + "theoretical_loss": 3.668687544733822, + "tokens_seen": 945815552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036033099297893683, + "loss": 2.6261, + "theoretical_loss": 3.668663104174012, + "tokens_seen": 945881088 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360320962888666, + "loss": 2.662, + "theoretical_loss": 3.668638665781633, + "tokens_seen": 945946624 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036031093279839525, + "loss": 2.6121, + "theoretical_loss": 3.6686142295563426, + "tokens_seen": 946012160 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036030090270812437, + "loss": 2.791, + "theoretical_loss": 3.668589795497799, + "tokens_seen": 946077696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602908726178536, + "loss": 2.6844, + "theoretical_loss": 3.6685653636056585, + "tokens_seen": 946143232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036028084252758273, + "loss": 2.9634, + "theoretical_loss": 3.66854093387958, + "tokens_seen": 946208768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036027081243731197, + "loss": 2.7207, + "theoretical_loss": 3.668516506319222, + "tokens_seen": 946274304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036026078234704115, + "loss": 2.5772, + "theoretical_loss": 3.668492080924241, + "tokens_seen": 946339840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036025075225677033, + "loss": 2.6456, + "theoretical_loss": 3.668467657694297, + "tokens_seen": 946405376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602407221664995, + "loss": 2.7981, + "theoretical_loss": 3.668443236629047, + "tokens_seen": 946470912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602306920762287, + "loss": 2.8633, + "theoretical_loss": 3.668418817728149, + "tokens_seen": 946536448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602206619859579, + "loss": 2.505, + "theoretical_loss": 3.6683944009912617, + "tokens_seen": 946601984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602106318956871, + "loss": 2.7682, + "theoretical_loss": 3.6683699864180443, + "tokens_seen": 946667520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036020060180541624, + "loss": 2.6689, + "theoretical_loss": 3.6683455740081543, + "tokens_seen": 946733056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601905717151455, + "loss": 2.7219, + "theoretical_loss": 3.6683211637612505, + "tokens_seen": 946798592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601805416248746, + "loss": 2.8951, + "theoretical_loss": 3.668296755676992, + "tokens_seen": 946864128 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036017051153460384, + "loss": 2.8538, + "theoretical_loss": 3.668272349755037, + "tokens_seen": 946929664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 529862, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7047557830810547, + "objective/train/theoretical_loss": 3.6682479459950446, + "objective/train/tokens_used": 967455200, + "theoretical_loss": 3.6682479459950446, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360160481444333, + "loss": 2.571, + "theoretical_loss": 3.6682479459950446, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601504513540622, + "loss": 2.64, + "theoretical_loss": 3.6682235443966733, + "tokens_seen": 947060736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601404212637914, + "loss": 2.7239, + "theoretical_loss": 3.6681991449595825, + "tokens_seen": 947126272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601303911735206, + "loss": 2.706, + "theoretical_loss": 3.668174747683431, + "tokens_seen": 947191808 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036012036108324974, + "loss": 2.7165, + "theoretical_loss": 3.6681503525678787, + "tokens_seen": 947257344 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360110330992979, + "loss": 2.756, + "theoretical_loss": 3.668125959612584, + "tokens_seen": 947322880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601003009027081, + "loss": 2.5381, + "theoretical_loss": 3.668101568817206, + "tokens_seen": 947388416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036009027081243734, + "loss": 2.6841, + "theoretical_loss": 3.6680771801814043, + "tokens_seen": 947453952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600802407221665, + "loss": 2.7336, + "theoretical_loss": 3.668052793704838, + "tokens_seen": 947519488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600702106318957, + "loss": 2.679, + "theoretical_loss": 3.668028409387168, + "tokens_seen": 947585024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600601805416249, + "loss": 2.7866, + "theoretical_loss": 3.668004027228052, + "tokens_seen": 947650560 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036005015045135406, + "loss": 2.6878, + "theoretical_loss": 3.667979647227151, + "tokens_seen": 947716096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036004012036108324, + "loss": 2.6239, + "theoretical_loss": 3.6679552693841244, + "tokens_seen": 947781632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600300902708125, + "loss": 2.6908, + "theoretical_loss": 3.6679308936986317, + "tokens_seen": 947847168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600200601805416, + "loss": 2.5497, + "theoretical_loss": 3.667906520170333, + "tokens_seen": 947912704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036001003009027084, + "loss": 2.7513, + "theoretical_loss": 3.6678821487988884, + "tokens_seen": 947978240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035999999999999997, + "loss": 2.3677, + "theoretical_loss": 3.667857779583958, + "tokens_seen": 948043776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599899699097292, + "loss": 2.6004, + "theoretical_loss": 3.6678334125252015, + "tokens_seen": 948109312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599799398194584, + "loss": 2.6877, + "theoretical_loss": 3.667809047622279, + "tokens_seen": 948174848 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035996990972918757, + "loss": 2.8092, + "theoretical_loss": 3.667784684874851, + "tokens_seen": 948240384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035995987963891675, + "loss": 2.6486, + "theoretical_loss": 3.667760324282579, + "tokens_seen": 948305920 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359949849548646, + "loss": 2.5051, + "theoretical_loss": 3.6677359658451216, + "tokens_seen": 948371456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599398194583751, + "loss": 2.7112, + "theoretical_loss": 3.6677116095621405, + "tokens_seen": 948436992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035992978936810434, + "loss": 2.5369, + "theoretical_loss": 3.6676872554332958, + "tokens_seen": 948502528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035991975927783347, + "loss": 2.6575, + "theoretical_loss": 3.667662903458248, + "tokens_seen": 948568064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 530380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.964254140853882, + "objective/train/theoretical_loss": 3.6676385536366585, + "objective/train/tokens_used": 969093600, + "theoretical_loss": 3.6676385536366585, + "tokens_seen": 948633600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599097291875627, + "loss": 2.7343, + "theoretical_loss": 3.6676385536366585, + "tokens_seen": 948633600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598996990972919, + "loss": 2.7499, + "theoretical_loss": 3.6676142059681878, + "tokens_seen": 948699136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035988966900702107, + "loss": 2.8896, + "theoretical_loss": 3.6675898604524964, + "tokens_seen": 948764672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035987963891675025, + "loss": 2.9244, + "theoretical_loss": 3.667565517089246, + "tokens_seen": 948830208 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035986960882647943, + "loss": 2.5677, + "theoretical_loss": 3.667541175878097, + "tokens_seen": 948895744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598595787362086, + "loss": 2.8888, + "theoretical_loss": 3.6675168368187103, + "tokens_seen": 948961280 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035984954864593785, + "loss": 2.5331, + "theoretical_loss": 3.6674924999107485, + "tokens_seen": 949026816 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359839518555667, + "loss": 2.8319, + "theoretical_loss": 3.6674681651538714, + "tokens_seen": 949092352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598294884653962, + "loss": 2.6815, + "theoretical_loss": 3.6674438325477405, + "tokens_seen": 949157888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035981945837512534, + "loss": 2.7486, + "theoretical_loss": 3.6674195020920184, + "tokens_seen": 949223424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035980942828485457, + "loss": 2.7056, + "theoretical_loss": 3.6673951737863657, + "tokens_seen": 949288960 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035979939819458375, + "loss": 2.6764, + "theoretical_loss": 3.667370847630444, + "tokens_seen": 949354496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035978936810431293, + "loss": 2.5544, + "theoretical_loss": 3.667346523623915, + "tokens_seen": 949420032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597793380140421, + "loss": 2.6938, + "theoretical_loss": 3.6673222017664404, + "tokens_seen": 949485568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035976930792377135, + "loss": 2.4797, + "theoretical_loss": 3.6672978820576825, + "tokens_seen": 949551104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597592778335005, + "loss": 2.8528, + "theoretical_loss": 3.667273564497303, + "tokens_seen": 949616640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597492477432297, + "loss": 2.6105, + "theoretical_loss": 3.6672492490849633, + "tokens_seen": 949682176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035973921765295884, + "loss": 2.5964, + "theoretical_loss": 3.667224935820326, + "tokens_seen": 949747712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597291875626881, + "loss": 2.7149, + "theoretical_loss": 3.667200624703053, + "tokens_seen": 949813248 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035971915747241726, + "loss": 2.5913, + "theoretical_loss": 3.6671763157328066, + "tokens_seen": 949878784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035970912738214644, + "loss": 2.6368, + "theoretical_loss": 3.667152008909249, + "tokens_seen": 949944320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596990972918756, + "loss": 2.7971, + "theoretical_loss": 3.6671277042320423, + "tokens_seen": 950009856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596890672016048, + "loss": 2.7024, + "theoretical_loss": 3.6671034017008495, + "tokens_seen": 950075392 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359679037111334, + "loss": 2.5597, + "theoretical_loss": 3.667079101315333, + "tokens_seen": 950140928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596690070210632, + "loss": 2.7964, + "theoretical_loss": 3.667054803075155, + "tokens_seen": 950206464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 531473, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.32260799407959, + "objective/train/theoretical_loss": 3.6670305069799785, + "objective/train/tokens_used": 970732000, + "theoretical_loss": 3.6670305069799785, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035965897693079234, + "loss": 2.6143, + "theoretical_loss": 3.6670305069799785, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596489468405216, + "loss": 2.657, + "theoretical_loss": 3.667006213029466, + "tokens_seen": 950337536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035963891675025076, + "loss": 2.7829, + "theoretical_loss": 3.6669819212232806, + "tokens_seen": 950403072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035962888665997994, + "loss": 2.7387, + "theoretical_loss": 3.6669576315610843, + "tokens_seen": 950468608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596188565697091, + "loss": 2.7885, + "theoretical_loss": 3.6669333440425413, + "tokens_seen": 950534144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596088264794383, + "loss": 2.854, + "theoretical_loss": 3.666909058667314, + "tokens_seen": 950599680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035959879638916754, + "loss": 2.5672, + "theoretical_loss": 3.666884775435066, + "tokens_seen": 950665216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595887662988967, + "loss": 2.4754, + "theoretical_loss": 3.6668604943454595, + "tokens_seen": 950730752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595787362086259, + "loss": 2.856, + "theoretical_loss": 3.6668362153981584, + "tokens_seen": 950796288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595687061183551, + "loss": 2.6182, + "theoretical_loss": 3.6668119385928257, + "tokens_seen": 950861824 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035955867602808426, + "loss": 2.7393, + "theoretical_loss": 3.6667876639291257, + "tokens_seen": 950927360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035954864593781344, + "loss": 2.5813, + "theoretical_loss": 3.6667633914067213, + "tokens_seen": 950992896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595386158475427, + "loss": 2.7133, + "theoretical_loss": 3.6667391210252758, + "tokens_seen": 951058432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595285857572718, + "loss": 2.5476, + "theoretical_loss": 3.666714852784453, + "tokens_seen": 951123968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035951855566700104, + "loss": 2.8157, + "theoretical_loss": 3.6666905866839175, + "tokens_seen": 951189504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035950852557673017, + "loss": 2.8043, + "theoretical_loss": 3.6666663227233314, + "tokens_seen": 951255040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594984954864594, + "loss": 2.7221, + "theoretical_loss": 3.66664206090236, + "tokens_seen": 951320576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594884653961886, + "loss": 2.758, + "theoretical_loss": 3.6666178012206663, + "tokens_seen": 951386112 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035947843530591777, + "loss": 2.7675, + "theoretical_loss": 3.666593543677915, + "tokens_seen": 951451648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035946840521564695, + "loss": 2.677, + "theoretical_loss": 3.66656928827377, + "tokens_seen": 951517184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594583751253762, + "loss": 2.8365, + "theoretical_loss": 3.6665450350078954, + "tokens_seen": 951582720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594483450351053, + "loss": 2.7003, + "theoretical_loss": 3.666520783879956, + "tokens_seen": 951648256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035943831494483454, + "loss": 2.7854, + "theoretical_loss": 3.6664965348896157, + "tokens_seen": 951713792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035942828485456367, + "loss": 2.5429, + "theoretical_loss": 3.6664722880365384, + "tokens_seen": 951779328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594182547642929, + "loss": 2.8141, + "theoretical_loss": 3.666448043320389, + "tokens_seen": 951844864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 532856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.025040864944458, + "objective/train/theoretical_loss": 3.666423800740832, + "objective/train/tokens_used": 972370400, + "theoretical_loss": 3.666423800740832, + "tokens_seen": 951910400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594082246740221, + "loss": 2.7602, + "theoretical_loss": 3.666423800740832, + "tokens_seen": 951910400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035939819458375127, + "loss": 2.6914, + "theoretical_loss": 3.666399560297532, + "tokens_seen": 951975936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035938816449348045, + "loss": 2.6439, + "theoretical_loss": 3.6663753219901545, + "tokens_seen": 952041472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035937813440320963, + "loss": 2.8814, + "theoretical_loss": 3.6663510858183628, + "tokens_seen": 952107008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593681043129388, + "loss": 2.5812, + "theoretical_loss": 3.6663268517818235, + "tokens_seen": 952172544 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035935807422266805, + "loss": 2.9106, + "theoretical_loss": 3.6663026198802, + "tokens_seen": 952238080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593480441323972, + "loss": 2.9617, + "theoretical_loss": 3.666278390113158, + "tokens_seen": 952303616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593380140421264, + "loss": 2.798, + "theoretical_loss": 3.6662541624803624, + "tokens_seen": 952369152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035932798395185554, + "loss": 2.7121, + "theoretical_loss": 3.6662299369814786, + "tokens_seen": 952434688 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035931795386158477, + "loss": 2.6545, + "theoretical_loss": 3.6662057136161716, + "tokens_seen": 952500224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035930792377131395, + "loss": 2.729, + "theoretical_loss": 3.6661814923841067, + "tokens_seen": 952565760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035929789368104313, + "loss": 2.8024, + "theoretical_loss": 3.6661572732849494, + "tokens_seen": 952631296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592878635907723, + "loss": 2.5495, + "theoretical_loss": 3.6661330563183654, + "tokens_seen": 952696832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035927783350050155, + "loss": 2.4999, + "theoretical_loss": 3.66610884148402, + "tokens_seen": 952762368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592678034102307, + "loss": 2.8948, + "theoretical_loss": 3.6660846287815794, + "tokens_seen": 952827904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592577733199599, + "loss": 2.654, + "theoretical_loss": 3.6660604182107077, + "tokens_seen": 952893440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035924774322968904, + "loss": 2.8302, + "theoretical_loss": 3.666036209771072, + "tokens_seen": 952958976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592377131394183, + "loss": 2.8485, + "theoretical_loss": 3.6660120034623382, + "tokens_seen": 953024512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035922768304914746, + "loss": 2.7843, + "theoretical_loss": 3.665987799284171, + "tokens_seen": 953090048 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035921765295887664, + "loss": 2.351, + "theoretical_loss": 3.665963597236238, + "tokens_seen": 953155584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592076228686058, + "loss": 2.6998, + "theoretical_loss": 3.6659393973182044, + "tokens_seen": 953221120 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359197592778335, + "loss": 2.5819, + "theoretical_loss": 3.665915199529736, + "tokens_seen": 953286656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591875626880642, + "loss": 2.6821, + "theoretical_loss": 3.6658910038704997, + "tokens_seen": 953352192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591775325977934, + "loss": 2.7457, + "theoretical_loss": 3.665866810340162, + "tokens_seen": 953417728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035916750250752254, + "loss": 2.6965, + "theoretical_loss": 3.665842618938388, + "tokens_seen": 953483264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 533452, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.747199773788452, + "objective/train/theoretical_loss": 3.6658184296648457, + "objective/train/tokens_used": 974008800, + "theoretical_loss": 3.6658184296648457, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591574724172518, + "loss": 2.8721, + "theoretical_loss": 3.6658184296648457, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035914744232698096, + "loss": 2.6881, + "theoretical_loss": 3.6657942425192003, + "tokens_seen": 953614336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035913741223671014, + "loss": 2.6932, + "theoretical_loss": 3.665770057501119, + "tokens_seen": 953679872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591273821464393, + "loss": 2.7589, + "theoretical_loss": 3.665745874610269, + "tokens_seen": 953745408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591173520561685, + "loss": 2.8227, + "theoretical_loss": 3.665721693846316, + "tokens_seen": 953810944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591073219658977, + "loss": 2.6922, + "theoretical_loss": 3.6656975152089277, + "tokens_seen": 953876480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590972918756269, + "loss": 2.6638, + "theoretical_loss": 3.6656733386977702, + "tokens_seen": 953942016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035908726178535605, + "loss": 2.9033, + "theoretical_loss": 3.665649164312511, + "tokens_seen": 954007552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590772316950853, + "loss": 2.6242, + "theoretical_loss": 3.665624992052817, + "tokens_seen": 954073088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590672016048144, + "loss": 2.7591, + "theoretical_loss": 3.665600821918355, + "tokens_seen": 954138624 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035905717151454364, + "loss": 2.748, + "theoretical_loss": 3.6655766539087935, + "tokens_seen": 954204160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590471414242728, + "loss": 2.5731, + "theoretical_loss": 3.6655524880237977, + "tokens_seen": 954269696 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359037111334002, + "loss": 2.6532, + "theoretical_loss": 3.665528324263037, + "tokens_seen": 954335232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590270812437312, + "loss": 2.5441, + "theoretical_loss": 3.665504162626177, + "tokens_seen": 954400768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035901705115346037, + "loss": 2.5651, + "theoretical_loss": 3.6654800031128865, + "tokens_seen": 954466304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035900702106318955, + "loss": 2.7077, + "theoretical_loss": 3.665455845722833, + "tokens_seen": 954531840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589969909729188, + "loss": 2.6124, + "theoretical_loss": 3.6654316904556827, + "tokens_seen": 954597376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589869608826479, + "loss": 2.7877, + "theoretical_loss": 3.665407537311105, + "tokens_seen": 954662912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035897693079237715, + "loss": 2.6148, + "theoretical_loss": 3.6653833862887675, + "tokens_seen": 954728448 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035896690070210633, + "loss": 2.7657, + "theoretical_loss": 3.665359237388337, + "tokens_seen": 954793984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589568706118355, + "loss": 2.6572, + "theoretical_loss": 3.6653350906094824, + "tokens_seen": 954859520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589468405215647, + "loss": 2.5009, + "theoretical_loss": 3.6653109459518713, + "tokens_seen": 954925056 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035893681043129387, + "loss": 2.5484, + "theoretical_loss": 3.6652868034151718, + "tokens_seen": 954990592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035892678034102305, + "loss": 2.7428, + "theoretical_loss": 3.6652626629990523, + "tokens_seen": 955056128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589167502507523, + "loss": 2.4657, + "theoretical_loss": 3.665238524703181, + "tokens_seen": 955121664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 534564, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8826329708099365, + "objective/train/theoretical_loss": 3.6652143885272257, + "objective/train/tokens_used": 975647200, + "theoretical_loss": 3.6652143885272257, + "tokens_seen": 955187200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589067201604814, + "loss": 2.5385, + "theoretical_loss": 3.6652143885272257, + "tokens_seen": 955187200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035889669007021065, + "loss": 2.8421, + "theoretical_loss": 3.6651902544708554, + "tokens_seen": 955252736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588866599799398, + "loss": 2.7383, + "theoretical_loss": 3.665166122533739, + "tokens_seen": 955318272 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358876629889669, + "loss": 2.716, + "theoretical_loss": 3.6651419927155438, + "tokens_seen": 955383808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588665997993982, + "loss": 2.7816, + "theoretical_loss": 3.665117865015939, + "tokens_seen": 955449344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588565697091274, + "loss": 2.6429, + "theoretical_loss": 3.6650937394345933, + "tokens_seen": 955514880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588465396188566, + "loss": 2.7899, + "theoretical_loss": 3.665069615971176, + "tokens_seen": 955580416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035883650952858574, + "loss": 2.6999, + "theoretical_loss": 3.665045494625355, + "tokens_seen": 955645952 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035882647943831497, + "loss": 2.5648, + "theoretical_loss": 3.6650213753967997, + "tokens_seen": 955711488 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035881644934804415, + "loss": 2.669, + "theoretical_loss": 3.664997258285179, + "tokens_seen": 955777024 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035880641925777333, + "loss": 2.6379, + "theoretical_loss": 3.664973143290162, + "tokens_seen": 955842560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587963891675025, + "loss": 2.6819, + "theoretical_loss": 3.6649490304114183, + "tokens_seen": 955908096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035878635907723175, + "loss": 2.7801, + "theoretical_loss": 3.664924919648616, + "tokens_seen": 955973632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587763289869609, + "loss": 2.6382, + "theoretical_loss": 3.6649008110014254, + "tokens_seen": 956039168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587662988966901, + "loss": 2.8571, + "theoretical_loss": 3.664876704469515, + "tokens_seen": 956104704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035875626880641924, + "loss": 2.9089, + "theoretical_loss": 3.6648526000525554, + "tokens_seen": 956170240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587462387161485, + "loss": 2.8262, + "theoretical_loss": 3.6648284977502152, + "tokens_seen": 956235776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035873620862587766, + "loss": 2.8647, + "theoretical_loss": 3.664804397562164, + "tokens_seen": 956301312 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035872617853560684, + "loss": 2.8062, + "theoretical_loss": 3.664780299488072, + "tokens_seen": 956366848 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358716148445336, + "loss": 2.5454, + "theoretical_loss": 3.6647562035276082, + "tokens_seen": 956432384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587061183550652, + "loss": 2.7496, + "theoretical_loss": 3.6647321096804433, + "tokens_seen": 956497920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586960882647944, + "loss": 2.6876, + "theoretical_loss": 3.6647080179462463, + "tokens_seen": 956563456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586860581745236, + "loss": 2.9231, + "theoretical_loss": 3.6646839283246875, + "tokens_seen": 956628992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035867602808425274, + "loss": 2.6204, + "theoretical_loss": 3.664659840815437, + "tokens_seen": 956694528 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358665997993982, + "loss": 2.719, + "theoretical_loss": 3.664635755418165, + "tokens_seen": 956760064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 535115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7175190448760986, + "objective/train/theoretical_loss": 3.6646116721325415, + "objective/train/tokens_used": 977285600, + "theoretical_loss": 3.6646116721325415, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035865596790371116, + "loss": 2.5719, + "theoretical_loss": 3.6646116721325415, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035864593781344034, + "loss": 2.4455, + "theoretical_loss": 3.664587590958237, + "tokens_seen": 956891136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586359077231695, + "loss": 2.6658, + "theoretical_loss": 3.6645635118949214, + "tokens_seen": 956956672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586258776328987, + "loss": 2.3844, + "theoretical_loss": 3.664539434942265, + "tokens_seen": 957022208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586158475426279, + "loss": 2.6877, + "theoretical_loss": 3.664515360099939, + "tokens_seen": 957087744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586058174523571, + "loss": 2.7858, + "theoretical_loss": 3.664491287367613, + "tokens_seen": 957153280 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035859578736208625, + "loss": 2.8758, + "theoretical_loss": 3.664467216744959, + "tokens_seen": 957218816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585857572718155, + "loss": 2.7512, + "theoretical_loss": 3.664443148231647, + "tokens_seen": 957284352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585757271815446, + "loss": 2.9092, + "theoretical_loss": 3.6644190818273463, + "tokens_seen": 957349888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035856569709127384, + "loss": 2.679, + "theoretical_loss": 3.66439501753173, + "tokens_seen": 957415424 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358555667001003, + "loss": 2.9037, + "theoretical_loss": 3.6643709553444683, + "tokens_seen": 957480960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585456369107322, + "loss": 2.5161, + "theoretical_loss": 3.6643468952652314, + "tokens_seen": 957546496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585356068204614, + "loss": 2.5333, + "theoretical_loss": 3.664322837293691, + "tokens_seen": 957612032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035852557673019057, + "loss": 2.8931, + "theoretical_loss": 3.664298781429518, + "tokens_seen": 957677568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035851554663991975, + "loss": 2.9314, + "theoretical_loss": 3.6642747276723844, + "tokens_seen": 957743104 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358505516549649, + "loss": 2.8517, + "theoretical_loss": 3.66425067602196, + "tokens_seen": 957808640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584954864593781, + "loss": 2.5505, + "theoretical_loss": 3.6642266264779177, + "tokens_seen": 957874176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035848545636910735, + "loss": 2.3913, + "theoretical_loss": 3.6642025790399284, + "tokens_seen": 957939712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035847542627883653, + "loss": 2.8246, + "theoretical_loss": 3.6641785337076627, + "tokens_seen": 958005248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584653961885657, + "loss": 2.536, + "theoretical_loss": 3.6641544904807937, + "tokens_seen": 958070784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584553660982949, + "loss": 2.7839, + "theoretical_loss": 3.664130449358992, + "tokens_seen": 958136320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035844533600802407, + "loss": 2.7568, + "theoretical_loss": 3.6641064103419296, + "tokens_seen": 958201856 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035843530591775325, + "loss": 2.5894, + "theoretical_loss": 3.6640823734292782, + "tokens_seen": 958267392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584252758274825, + "loss": 2.809, + "theoretical_loss": 3.66405833862071, + "tokens_seen": 958332928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584152457372116, + "loss": 2.5561, + "theoretical_loss": 3.6640343059158966, + "tokens_seen": 958398464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 536290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.003551483154297, + "objective/train/theoretical_loss": 3.66401027531451, + "objective/train/tokens_used": 978924000, + "theoretical_loss": 3.66401027531451, + "tokens_seen": 958464000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035840521564694085, + "loss": 2.7357, + "theoretical_loss": 3.66401027531451, + "tokens_seen": 958464000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035839518555667, + "loss": 2.6729, + "theoretical_loss": 3.663986246816223, + "tokens_seen": 958529536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583851554663992, + "loss": 2.7778, + "theoretical_loss": 3.663962220420707, + "tokens_seen": 958595072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583751253761284, + "loss": 2.7285, + "theoretical_loss": 3.663938196127634, + "tokens_seen": 958660608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583650952858576, + "loss": 2.4119, + "theoretical_loss": 3.6639141739366776, + "tokens_seen": 958726144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035835506519558676, + "loss": 2.7283, + "theoretical_loss": 3.663890153847509, + "tokens_seen": 958791680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035834503510531594, + "loss": 2.4578, + "theoretical_loss": 3.663866135859801, + "tokens_seen": 958857216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583350050150451, + "loss": 2.7346, + "theoretical_loss": 3.663842119973226, + "tokens_seen": 958922752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035832497492477435, + "loss": 2.6133, + "theoretical_loss": 3.663818106187457, + "tokens_seen": 958988288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583149448345035, + "loss": 2.6151, + "theoretical_loss": 3.6637940945021663, + "tokens_seen": 959053824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583049147442327, + "loss": 2.6634, + "theoretical_loss": 3.663770084917027, + "tokens_seen": 959119360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582948846539619, + "loss": 2.7547, + "theoretical_loss": 3.663746077431712, + "tokens_seen": 959184896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582848545636911, + "loss": 2.8404, + "theoretical_loss": 3.6637220720458936, + "tokens_seen": 959250432 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035827482447342026, + "loss": 2.5344, + "theoretical_loss": 3.6636980687592455, + "tokens_seen": 959315968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035826479438314944, + "loss": 2.7036, + "theoretical_loss": 3.66367406757144, + "tokens_seen": 959381504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582547642928786, + "loss": 2.6886, + "theoretical_loss": 3.6636500684821507, + "tokens_seen": 959447040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035824473420260786, + "loss": 2.7948, + "theoretical_loss": 3.663626071491051, + "tokens_seen": 959512576 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358234704112337, + "loss": 2.8091, + "theoretical_loss": 3.6636020765978135, + "tokens_seen": 959578112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582246740220662, + "loss": 2.6147, + "theoretical_loss": 3.663578083802112, + "tokens_seen": 959643648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035821464393179535, + "loss": 2.6451, + "theoretical_loss": 3.6635540931036195, + "tokens_seen": 959709184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582046138415246, + "loss": 2.6816, + "theoretical_loss": 3.6635301045020103, + "tokens_seen": 959774720 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035819458375125376, + "loss": 2.7626, + "theoretical_loss": 3.663506117996958, + "tokens_seen": 959840256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035818455366098294, + "loss": 2.5617, + "theoretical_loss": 3.6634821335881345, + "tokens_seen": 959905792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581745235707121, + "loss": 2.6693, + "theoretical_loss": 3.663458151275216, + "tokens_seen": 959971328 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035816449348044136, + "loss": 2.64, + "theoretical_loss": 3.663434171057874, + "tokens_seen": 960036864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 537086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.786989688873291, + "objective/train/theoretical_loss": 3.6634101929357836, + "objective/train/tokens_used": 980562400, + "theoretical_loss": 3.6634101929357836, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581544633901705, + "loss": 2.581, + "theoretical_loss": 3.6634101929357836, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581444332998997, + "loss": 2.7225, + "theoretical_loss": 3.663386216908619, + "tokens_seen": 960167936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035813440320962885, + "loss": 2.5938, + "theoretical_loss": 3.6633622429760533, + "tokens_seen": 960233472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581243731193581, + "loss": 2.7526, + "theoretical_loss": 3.6633382711377607, + "tokens_seen": 960299008 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035811434302908727, + "loss": 2.7211, + "theoretical_loss": 3.663314301393416, + "tokens_seen": 960364544 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035810431293881645, + "loss": 2.4448, + "theoretical_loss": 3.663290333742693, + "tokens_seen": 960430080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580942828485457, + "loss": 2.5715, + "theoretical_loss": 3.6632663681852655, + "tokens_seen": 960495616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580842527582748, + "loss": 2.6514, + "theoretical_loss": 3.663242404720809, + "tokens_seen": 960561152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035807422266800404, + "loss": 2.6498, + "theoretical_loss": 3.663218443348997, + "tokens_seen": 960626688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580641925777332, + "loss": 2.7838, + "theoretical_loss": 3.663194484069505, + "tokens_seen": 960692224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580541624874624, + "loss": 2.4934, + "theoretical_loss": 3.6631705268820065, + "tokens_seen": 960757760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580441323971916, + "loss": 2.6763, + "theoretical_loss": 3.6631465717861764, + "tokens_seen": 960823296 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035803410230692077, + "loss": 2.7096, + "theoretical_loss": 3.6631226187816894, + "tokens_seen": 960888832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035802407221664995, + "loss": 2.7574, + "theoretical_loss": 3.663098667868221, + "tokens_seen": 960954368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580140421263792, + "loss": 2.3962, + "theoretical_loss": 3.6630747190454453, + "tokens_seen": 961019904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580040120361083, + "loss": 2.5524, + "theoretical_loss": 3.6630507723130377, + "tokens_seen": 961085440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035799398194583755, + "loss": 2.7353, + "theoretical_loss": 3.663026827670673, + "tokens_seen": 961150976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035798395185556673, + "loss": 2.6363, + "theoretical_loss": 3.6630028851180265, + "tokens_seen": 961216512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579739217652959, + "loss": 2.5274, + "theoretical_loss": 3.662978944654773, + "tokens_seen": 961282048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579638916750251, + "loss": 2.7494, + "theoretical_loss": 3.662955006280588, + "tokens_seen": 961347584 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035795386158475427, + "loss": 2.7442, + "theoretical_loss": 3.662931069995147, + "tokens_seen": 961413120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035794383149448345, + "loss": 2.8181, + "theoretical_loss": 3.6629071357981244, + "tokens_seen": 961478656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579338014042127, + "loss": 2.6398, + "theoretical_loss": 3.662883203689197, + "tokens_seen": 961544192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579237713139418, + "loss": 2.8939, + "theoretical_loss": 3.6628592736680394, + "tokens_seen": 961609728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035791374122367105, + "loss": 2.6428, + "theoretical_loss": 3.6628353457343277, + "tokens_seen": 961675264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 537653, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.703756809234619, + "objective/train/theoretical_loss": 3.6628114198877375, + "objective/train/tokens_used": 982200800, + "theoretical_loss": 3.6628114198877375, + "tokens_seen": 961740800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579037111334002, + "loss": 2.5893, + "theoretical_loss": 3.6628114198877375, + "tokens_seen": 961740800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578936810431294, + "loss": 2.5064, + "theoretical_loss": 3.6627874961279447, + "tokens_seen": 961806336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578836509528586, + "loss": 2.6169, + "theoretical_loss": 3.6627635744546243, + "tokens_seen": 961871872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578736208625878, + "loss": 2.5647, + "theoretical_loss": 3.662739654867453, + "tokens_seen": 961937408 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035786359077231696, + "loss": 2.7298, + "theoretical_loss": 3.662715737366107, + "tokens_seen": 962002944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035785356068204614, + "loss": 2.5342, + "theoretical_loss": 3.6626918219502613, + "tokens_seen": 962068480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578435305917753, + "loss": 2.5674, + "theoretical_loss": 3.6626679086195923, + "tokens_seen": 962134016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035783350050150455, + "loss": 2.5954, + "theoretical_loss": 3.6626439973737765, + "tokens_seen": 962199552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578234704112337, + "loss": 2.8665, + "theoretical_loss": 3.6626200882124906, + "tokens_seen": 962265088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578134403209629, + "loss": 2.7817, + "theoretical_loss": 3.6625961811354104, + "tokens_seen": 962330624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578034102306921, + "loss": 2.4717, + "theoretical_loss": 3.662572276142212, + "tokens_seen": 962396160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577933801404213, + "loss": 2.6832, + "theoretical_loss": 3.662548373232572, + "tokens_seen": 962461696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035778335005015046, + "loss": 2.5416, + "theoretical_loss": 3.662524472406168, + "tokens_seen": 962527232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035777331995987964, + "loss": 2.7669, + "theoretical_loss": 3.662500573662675, + "tokens_seen": 962592768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577632898696088, + "loss": 2.6586, + "theoretical_loss": 3.66247667700177, + "tokens_seen": 962658304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035775325977933806, + "loss": 2.4959, + "theoretical_loss": 3.662452782423131, + "tokens_seen": 962723840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577432296890672, + "loss": 2.7372, + "theoretical_loss": 3.662428889926434, + "tokens_seen": 962789376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577331995987964, + "loss": 2.9573, + "theoretical_loss": 3.662404999511356, + "tokens_seen": 962854912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035772316950852555, + "loss": 2.5443, + "theoretical_loss": 3.6623811111775737, + "tokens_seen": 962920448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577131394182548, + "loss": 2.7655, + "theoretical_loss": 3.6623572249247642, + "tokens_seen": 962985984 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035770310932798396, + "loss": 2.6151, + "theoretical_loss": 3.662333340752604, + "tokens_seen": 963051520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035769307923771314, + "loss": 2.7865, + "theoretical_loss": 3.662309458660772, + "tokens_seen": 963117056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576830491474423, + "loss": 2.7037, + "theoretical_loss": 3.6622855786489446, + "tokens_seen": 963182592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035767301905717156, + "loss": 2.4568, + "theoretical_loss": 3.6622617007167992, + "tokens_seen": 963248128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576629889669007, + "loss": 2.5051, + "theoretical_loss": 3.6622378248640124, + "tokens_seen": 963313664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 538877, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.671361207962036, + "objective/train/theoretical_loss": 3.6622139510902625, + "objective/train/tokens_used": 983839200, + "theoretical_loss": 3.6622139510902625, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576529588766299, + "loss": 2.464, + "theoretical_loss": 3.6622139510902625, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035764292878635905, + "loss": 2.8269, + "theoretical_loss": 3.662190079395227, + "tokens_seen": 963444736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576328986960883, + "loss": 2.7938, + "theoretical_loss": 3.6621662097785834, + "tokens_seen": 963510272 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035762286860581747, + "loss": 2.6486, + "theoretical_loss": 3.66214234224001, + "tokens_seen": 963575808 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035761283851554665, + "loss": 2.713, + "theoretical_loss": 3.662118476779183, + "tokens_seen": 963641344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035760280842527583, + "loss": 2.7186, + "theoretical_loss": 3.662094613395782, + "tokens_seen": 963706880 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357592778335005, + "loss": 2.4699, + "theoretical_loss": 3.6620707520894835, + "tokens_seen": 963772416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575827482447342, + "loss": 2.7424, + "theoretical_loss": 3.662046892859966, + "tokens_seen": 963837952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575727181544634, + "loss": 2.6724, + "theoretical_loss": 3.662023035706908, + "tokens_seen": 963903488 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035756268806419255, + "loss": 2.6567, + "theoretical_loss": 3.661999180629987, + "tokens_seen": 963969024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575526579739218, + "loss": 2.5726, + "theoretical_loss": 3.6619753276288822, + "tokens_seen": 964034560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575426278836509, + "loss": 2.6506, + "theoretical_loss": 3.661951476703271, + "tokens_seen": 964100096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035753259779338015, + "loss": 2.6112, + "theoretical_loss": 3.6619276278528314, + "tokens_seen": 964165632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035752256770310933, + "loss": 2.6589, + "theoretical_loss": 3.6619037810772426, + "tokens_seen": 964231168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575125376128385, + "loss": 2.7243, + "theoretical_loss": 3.6618799363761827, + "tokens_seen": 964296704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575025075225677, + "loss": 2.7628, + "theoretical_loss": 3.6618560937493303, + "tokens_seen": 964362240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035749247743229693, + "loss": 2.6175, + "theoretical_loss": 3.661832253196364, + "tokens_seen": 964427776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035748244734202606, + "loss": 2.6583, + "theoretical_loss": 3.6618084147169627, + "tokens_seen": 964493312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574724172517553, + "loss": 2.9021, + "theoretical_loss": 3.6617845783108054, + "tokens_seen": 964558848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574623871614844, + "loss": 2.653, + "theoretical_loss": 3.6617607439775703, + "tokens_seen": 964624384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035745235707121365, + "loss": 2.4431, + "theoretical_loss": 3.661736911716936, + "tokens_seen": 964689920 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035744232698094283, + "loss": 2.6706, + "theoretical_loss": 3.6617130815285828, + "tokens_seen": 964755456 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357432296890672, + "loss": 2.6288, + "theoretical_loss": 3.661689253412189, + "tokens_seen": 964820992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574222668004012, + "loss": 2.7101, + "theoretical_loss": 3.661665427367434, + "tokens_seen": 964886528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574122367101304, + "loss": 2.7293, + "theoretical_loss": 3.6616416033939965, + "tokens_seen": 964952064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 539426, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4708588123321533, + "objective/train/theoretical_loss": 3.6616177814915556, + "objective/train/tokens_used": 985477600, + "theoretical_loss": 3.6616177814915556, + "tokens_seen": 965017600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035740220661985956, + "loss": 2.766, + "theoretical_loss": 3.6616177814915556, + "tokens_seen": 965017600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573921765295888, + "loss": 2.9734, + "theoretical_loss": 3.6615939616597917, + "tokens_seen": 965083136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573821464393179, + "loss": 2.5102, + "theoretical_loss": 3.661570143898383, + "tokens_seen": 965148672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035737211634904716, + "loss": 2.6679, + "theoretical_loss": 3.6615463282070104, + "tokens_seen": 965214208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573620862587763, + "loss": 2.6167, + "theoretical_loss": 3.6615225145853523, + "tokens_seen": 965279744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573520561685055, + "loss": 2.6641, + "theoretical_loss": 3.6614987030330886, + "tokens_seen": 965345280 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035734202607823475, + "loss": 2.8822, + "theoretical_loss": 3.661474893549899, + "tokens_seen": 965410816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573319959879639, + "loss": 2.8572, + "theoretical_loss": 3.6614510861354637, + "tokens_seen": 965476352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573219658976931, + "loss": 2.6992, + "theoretical_loss": 3.661427280789462, + "tokens_seen": 965541888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573119358074223, + "loss": 2.5701, + "theoretical_loss": 3.6614034775115742, + "tokens_seen": 965607424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573019057171515, + "loss": 2.788, + "theoretical_loss": 3.66137967630148, + "tokens_seen": 965672960 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035729187562688066, + "loss": 2.9269, + "theoretical_loss": 3.6613558771588592, + "tokens_seen": 965738496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035728184553660984, + "loss": 2.7455, + "theoretical_loss": 3.6613320800833926, + "tokens_seen": 965804032 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357271815446339, + "loss": 2.7057, + "theoretical_loss": 3.6613082850747602, + "tokens_seen": 965869568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035726178535606826, + "loss": 2.7845, + "theoretical_loss": 3.6612844921326415, + "tokens_seen": 965935104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572517552657974, + "loss": 2.6283, + "theoretical_loss": 3.661260701256718, + "tokens_seen": 966000640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572417251755266, + "loss": 2.5378, + "theoretical_loss": 3.66123691244667, + "tokens_seen": 966066176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035723169508525575, + "loss": 2.626, + "theoretical_loss": 3.6612131257021767, + "tokens_seen": 966131712 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357221664994985, + "loss": 2.6145, + "theoretical_loss": 3.6611893410229204, + "tokens_seen": 966197248 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035721163490471416, + "loss": 2.5952, + "theoretical_loss": 3.6611655584085803, + "tokens_seen": 966262784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035720160481444334, + "loss": 2.3264, + "theoretical_loss": 3.6611417778588375, + "tokens_seen": 966328320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571915747241725, + "loss": 2.5936, + "theoretical_loss": 3.661117999373374, + "tokens_seen": 966393856 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035718154463390176, + "loss": 2.7259, + "theoretical_loss": 3.661094222951869, + "tokens_seen": 966459392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571715145436309, + "loss": 2.6951, + "theoretical_loss": 3.6610704485940033, + "tokens_seen": 966524928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571614844533601, + "loss": 2.7819, + "theoretical_loss": 3.661046676299459, + "tokens_seen": 966590464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 540676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.884268283843994, + "objective/train/theoretical_loss": 3.6610229060679167, + "objective/train/tokens_used": 987116000, + "theoretical_loss": 3.6610229060679167, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035715145436308925, + "loss": 2.5583, + "theoretical_loss": 3.6610229060679167, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571414242728185, + "loss": 2.6991, + "theoretical_loss": 3.660999137899058, + "tokens_seen": 966721536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035713139418254767, + "loss": 2.7206, + "theoretical_loss": 3.660975371792563, + "tokens_seen": 966787072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035712136409227685, + "loss": 2.681, + "theoretical_loss": 3.660951607748114, + "tokens_seen": 966852608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035711133400200603, + "loss": 2.5881, + "theoretical_loss": 3.660927845765392, + "tokens_seen": 966918144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571013039117352, + "loss": 2.7071, + "theoretical_loss": 3.660904085844078, + "tokens_seen": 966983680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570912738214644, + "loss": 2.7228, + "theoretical_loss": 3.660880327983854, + "tokens_seen": 967049216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570812437311936, + "loss": 2.6058, + "theoretical_loss": 3.660856572184401, + "tokens_seen": 967114752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035707121364092275, + "loss": 2.5509, + "theoretical_loss": 3.6608328184454013, + "tokens_seen": 967180288 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357061183550652, + "loss": 2.6556, + "theoretical_loss": 3.6608090667665363, + "tokens_seen": 967245824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570511534603811, + "loss": 2.796, + "theoretical_loss": 3.6607853171474876, + "tokens_seen": 967311360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035704112337011035, + "loss": 2.6946, + "theoretical_loss": 3.660761569587937, + "tokens_seen": 967376896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035703109327983953, + "loss": 2.5267, + "theoretical_loss": 3.660737824087567, + "tokens_seen": 967442432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570210631895687, + "loss": 2.7813, + "theoretical_loss": 3.660714080646059, + "tokens_seen": 967507968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570110330992979, + "loss": 2.6722, + "theoretical_loss": 3.660690339263095, + "tokens_seen": 967573504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035700100300902713, + "loss": 2.8932, + "theoretical_loss": 3.6606665999383576, + "tokens_seen": 967639040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035699097291875626, + "loss": 2.636, + "theoretical_loss": 3.660642862671528, + "tokens_seen": 967704576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569809428284855, + "loss": 2.7414, + "theoretical_loss": 3.66061912746229, + "tokens_seen": 967770112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569709127382146, + "loss": 2.823, + "theoretical_loss": 3.6605953943103247, + "tokens_seen": 967835648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035696088264794385, + "loss": 2.8707, + "theoretical_loss": 3.6605716632153147, + "tokens_seen": 967901184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035695085255767303, + "loss": 2.7507, + "theoretical_loss": 3.660547934176943, + "tokens_seen": 967966720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569408224674022, + "loss": 2.9235, + "theoretical_loss": 3.660524207194891, + "tokens_seen": 968032256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569307923771314, + "loss": 2.5484, + "theoretical_loss": 3.660500482268843, + "tokens_seen": 968097792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569207622868606, + "loss": 2.7966, + "theoretical_loss": 3.66047675939848, + "tokens_seen": 968163328 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035691073219658976, + "loss": 2.7508, + "theoretical_loss": 3.660453038583486, + "tokens_seen": 968228864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 541190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9151451587677, + "objective/train/theoretical_loss": 3.660429319823543, + "objective/train/tokens_used": 988754400, + "theoretical_loss": 3.660429319823543, + "tokens_seen": 968294400 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356900702106319, + "loss": 2.8765, + "theoretical_loss": 3.660429319823543, + "tokens_seen": 968294400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568906720160481, + "loss": 2.9427, + "theoretical_loss": 3.6604056031183343, + "tokens_seen": 968359936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035688064192577736, + "loss": 2.8068, + "theoretical_loss": 3.6603818884675423, + "tokens_seen": 968425472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568706118355065, + "loss": 2.6032, + "theoretical_loss": 3.660358175870851, + "tokens_seen": 968491008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568605817452357, + "loss": 2.5964, + "theoretical_loss": 3.660334465327943, + "tokens_seen": 968556544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568505516549649, + "loss": 2.7074, + "theoretical_loss": 3.6603107568385016, + "tokens_seen": 968622080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568405215646941, + "loss": 2.8676, + "theoretical_loss": 3.66028705040221, + "tokens_seen": 968687616 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035683049147442326, + "loss": 2.6544, + "theoretical_loss": 3.6602633460187506, + "tokens_seen": 968753152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568204613841525, + "loss": 2.7562, + "theoretical_loss": 3.6602396436878077, + "tokens_seen": 968818688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568104312938816, + "loss": 2.3719, + "theoretical_loss": 3.6602159434090655, + "tokens_seen": 968884224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035680040120361086, + "loss": 2.7534, + "theoretical_loss": 3.660192245182206, + "tokens_seen": 968949760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035679037111334, + "loss": 2.534, + "theoretical_loss": 3.6601685490069134, + "tokens_seen": 969015296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567803410230692, + "loss": 2.62, + "theoretical_loss": 3.660144854882872, + "tokens_seen": 969080832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567703109327984, + "loss": 2.8146, + "theoretical_loss": 3.660121162809764, + "tokens_seen": 969146368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567602808425276, + "loss": 2.7872, + "theoretical_loss": 3.6600974727872746, + "tokens_seen": 969211904 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035675025075225677, + "loss": 2.7477, + "theoretical_loss": 3.6600737848150873, + "tokens_seen": 969277440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035674022066198595, + "loss": 2.8665, + "theoretical_loss": 3.660050098892886, + "tokens_seen": 969342976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035673019057171513, + "loss": 2.9426, + "theoretical_loss": 3.660026415020355, + "tokens_seen": 969408512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035672016048144436, + "loss": 2.5801, + "theoretical_loss": 3.6600027331971776, + "tokens_seen": 969474048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567101303911735, + "loss": 2.5656, + "theoretical_loss": 3.6599790534230383, + "tokens_seen": 969539584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567001003009027, + "loss": 2.6174, + "theoretical_loss": 3.659955375697622, + "tokens_seen": 969605120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035669007021063185, + "loss": 2.5564, + "theoretical_loss": 3.659931700020612, + "tokens_seen": 969670656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566800401203611, + "loss": 2.4917, + "theoretical_loss": 3.659908026391693, + "tokens_seen": 969736192 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035667001003009027, + "loss": 2.6846, + "theoretical_loss": 3.6598843548105497, + "tokens_seen": 969801728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035665997993981945, + "loss": 2.7662, + "theoretical_loss": 3.6598606852768665, + "tokens_seen": 969867264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 542499, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.745290517807007, + "objective/train/theoretical_loss": 3.659837017790328, + "objective/train/tokens_used": 990392800, + "theoretical_loss": 3.659837017790328, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035664994984954863, + "loss": 2.8151, + "theoretical_loss": 3.659837017790328, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035663991975927787, + "loss": 2.6502, + "theoretical_loss": 3.6598133523506187, + "tokens_seen": 969998336 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356629889669007, + "loss": 2.6904, + "theoretical_loss": 3.6597896889574235, + "tokens_seen": 970063872 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035661985957873623, + "loss": 2.7728, + "theoretical_loss": 3.6597660276104262, + "tokens_seen": 970129408 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035660982948846536, + "loss": 2.4748, + "theoretical_loss": 3.6597423683093133, + "tokens_seen": 970194944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565997993981946, + "loss": 2.4516, + "theoretical_loss": 3.6597187110537686, + "tokens_seen": 970260480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565897693079238, + "loss": 3.0593, + "theoretical_loss": 3.6596950558434775, + "tokens_seen": 970326016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035657973921765295, + "loss": 2.6859, + "theoretical_loss": 3.659671402678125, + "tokens_seen": 970391552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565697091273822, + "loss": 2.7334, + "theoretical_loss": 3.6596477515573964, + "tokens_seen": 970457088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565596790371113, + "loss": 2.5428, + "theoretical_loss": 3.6596241024809766, + "tokens_seen": 970522624 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035654964894684055, + "loss": 2.6518, + "theoretical_loss": 3.65960045544855, + "tokens_seen": 970588160 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035653961885656973, + "loss": 2.608, + "theoretical_loss": 3.6595768104598045, + "tokens_seen": 970653696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565295887662989, + "loss": 2.776, + "theoretical_loss": 3.659553167514423, + "tokens_seen": 970719232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565195586760281, + "loss": 2.8407, + "theoretical_loss": 3.659529526612092, + "tokens_seen": 970784768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035650952858575733, + "loss": 2.5555, + "theoretical_loss": 3.659505887752497, + "tokens_seen": 970850304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035649949849548646, + "loss": 2.8917, + "theoretical_loss": 3.6594822509353238, + "tokens_seen": 970915840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564894684052157, + "loss": 2.8816, + "theoretical_loss": 3.659458616160258, + "tokens_seen": 970981376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564794383149448, + "loss": 2.5581, + "theoretical_loss": 3.6594349834269844, + "tokens_seen": 971046912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035646940822467405, + "loss": 2.3331, + "theoretical_loss": 3.65941135273519, + "tokens_seen": 971112448 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035645937813440323, + "loss": 2.658, + "theoretical_loss": 3.6593877240845605, + "tokens_seen": 971177984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564493480441324, + "loss": 2.8178, + "theoretical_loss": 3.659364097474782, + "tokens_seen": 971243520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564393179538616, + "loss": 2.5768, + "theoretical_loss": 3.65934047290554, + "tokens_seen": 971309056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564292878635908, + "loss": 2.509, + "theoretical_loss": 3.6593168503765203, + "tokens_seen": 971374592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035641925777331996, + "loss": 2.5761, + "theoretical_loss": 3.6592932298874103, + "tokens_seen": 971440128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564092276830492, + "loss": 2.8889, + "theoretical_loss": 3.659269611437895, + "tokens_seen": 971505664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 543787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6714510917663574, + "objective/train/theoretical_loss": 3.6592459950276623, + "objective/train/tokens_used": 992031200, + "theoretical_loss": 3.6592459950276623, + "tokens_seen": 971571200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563991975927783, + "loss": 2.7689, + "theoretical_loss": 3.6592459950276623, + "tokens_seen": 971571200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035638916750250756, + "loss": 2.3759, + "theoretical_loss": 3.659222380656397, + "tokens_seen": 971636736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563791374122367, + "loss": 2.6477, + "theoretical_loss": 3.659198768323786, + "tokens_seen": 971702272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563691073219659, + "loss": 2.633, + "theoretical_loss": 3.659175158029516, + "tokens_seen": 971767808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563590772316951, + "loss": 2.8631, + "theoretical_loss": 3.6591515497732736, + "tokens_seen": 971833344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563490471414243, + "loss": 2.9037, + "theoretical_loss": 3.6591279435547452, + "tokens_seen": 971898880 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035633901705115346, + "loss": 2.6792, + "theoretical_loss": 3.6591043393736173, + "tokens_seen": 971964416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563289869608827, + "loss": 2.6147, + "theoretical_loss": 3.659080737229578, + "tokens_seen": 972029952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563189568706118, + "loss": 2.5756, + "theoretical_loss": 3.6590571371223133, + "tokens_seen": 972095488 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035630892678034106, + "loss": 2.5859, + "theoretical_loss": 3.6590335390515096, + "tokens_seen": 972161024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562988966900702, + "loss": 2.7238, + "theoretical_loss": 3.6590099430168546, + "tokens_seen": 972226560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562888665997994, + "loss": 2.7529, + "theoretical_loss": 3.6589863490180354, + "tokens_seen": 972292096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562788365095286, + "loss": 2.6947, + "theoretical_loss": 3.6589627570547387, + "tokens_seen": 972357632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562688064192578, + "loss": 2.8994, + "theoretical_loss": 3.6589391671266522, + "tokens_seen": 972423168 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035625877632898697, + "loss": 2.6734, + "theoretical_loss": 3.6589155792334633, + "tokens_seen": 972488704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035624874623871615, + "loss": 2.7424, + "theoretical_loss": 3.6588919933748585, + "tokens_seen": 972554240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035623871614844533, + "loss": 2.8223, + "theoretical_loss": 3.658868409550526, + "tokens_seen": 972619776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035622868605817456, + "loss": 2.7093, + "theoretical_loss": 3.6588448277601526, + "tokens_seen": 972685312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562186559679037, + "loss": 2.7391, + "theoretical_loss": 3.658821248003427, + "tokens_seen": 972750848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562086258776329, + "loss": 2.7881, + "theoretical_loss": 3.6587976702800353, + "tokens_seen": 972816384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035619859578736205, + "loss": 2.6558, + "theoretical_loss": 3.658774094589667, + "tokens_seen": 972881920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561885656970913, + "loss": 2.6641, + "theoretical_loss": 3.658750520932008, + "tokens_seen": 972947456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035617853560682047, + "loss": 2.6948, + "theoretical_loss": 3.6587269493067476, + "tokens_seen": 973012992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035616850551654965, + "loss": 2.6006, + "theoretical_loss": 3.6587033797135726, + "tokens_seen": 973078528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035615847542627883, + "loss": 2.5074, + "theoretical_loss": 3.658679812152172, + "tokens_seen": 973144064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 544242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9670348167419434, + "objective/train/theoretical_loss": 3.658656246622233, + "objective/train/tokens_used": 993669600, + "theoretical_loss": 3.658656246622233, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035614844533600807, + "loss": 2.8928, + "theoretical_loss": 3.658656246622233, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561384152457372, + "loss": 2.6655, + "theoretical_loss": 3.658632683123444, + "tokens_seen": 973275136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035612838515546643, + "loss": 2.6035, + "theoretical_loss": 3.658609121655494, + "tokens_seen": 973340672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035611835506519556, + "loss": 2.646, + "theoretical_loss": 3.65858556221807, + "tokens_seen": 973406208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561083249749248, + "loss": 2.6821, + "theoretical_loss": 3.6585620048108605, + "tokens_seen": 973471744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035609829488465397, + "loss": 2.8966, + "theoretical_loss": 3.6585384494335544, + "tokens_seen": 973537280 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035608826479438315, + "loss": 2.6911, + "theoretical_loss": 3.65851489608584, + "tokens_seen": 973602816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035607823470411233, + "loss": 2.7678, + "theoretical_loss": 3.658491344767406, + "tokens_seen": 973668352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560682046138415, + "loss": 2.8951, + "theoretical_loss": 3.6584677954779403, + "tokens_seen": 973733888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560581745235707, + "loss": 2.68, + "theoretical_loss": 3.6584442482171324, + "tokens_seen": 973799424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035604814443329993, + "loss": 2.8754, + "theoretical_loss": 3.658420702984671, + "tokens_seen": 973864960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035603811434302906, + "loss": 2.6871, + "theoretical_loss": 3.658397159780244, + "tokens_seen": 973930496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560280842527583, + "loss": 2.4913, + "theoretical_loss": 3.658373618603541, + "tokens_seen": 973996032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560180541624874, + "loss": 2.5447, + "theoretical_loss": 3.6583500794542503, + "tokens_seen": 974061568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035600802407221666, + "loss": 2.6892, + "theoretical_loss": 3.6583265423320617, + "tokens_seen": 974127104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035599799398194584, + "loss": 2.5806, + "theoretical_loss": 3.6583030072366647, + "tokens_seen": 974192640 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355987963891675, + "loss": 2.5213, + "theoretical_loss": 3.6582794741677467, + "tokens_seen": 974258176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559779338014042, + "loss": 2.6246, + "theoretical_loss": 3.658255943124998, + "tokens_seen": 974323712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035596790371113344, + "loss": 2.7462, + "theoretical_loss": 3.6582324141081086, + "tokens_seen": 974389248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035595787362086256, + "loss": 2.6027, + "theoretical_loss": 3.6582088871167664, + "tokens_seen": 974454784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559478435305918, + "loss": 2.6232, + "theoretical_loss": 3.6581853621506615, + "tokens_seen": 974520320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559378134403209, + "loss": 2.7216, + "theoretical_loss": 3.658161839209484, + "tokens_seen": 974585856 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035592778335005016, + "loss": 2.8452, + "theoretical_loss": 3.658138318292922, + "tokens_seen": 974651392 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035591775325977934, + "loss": 2.6781, + "theoretical_loss": 3.658114799400666, + "tokens_seen": 974716928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559077231695085, + "loss": 2.5703, + "theoretical_loss": 3.6580912825324057, + "tokens_seen": 974782464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 545526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5406079292297363, + "objective/train/theoretical_loss": 3.6580677676878306, + "objective/train/tokens_used": 995308000, + "theoretical_loss": 3.6580677676878306, + "tokens_seen": 974848000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558976930792377, + "loss": 2.6219, + "theoretical_loss": 3.6580677676878306, + "tokens_seen": 974848000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558876629889669, + "loss": 2.7312, + "theoretical_loss": 3.658044254866631, + "tokens_seen": 974913536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035587763289869607, + "loss": 2.6663, + "theoretical_loss": 3.6580207440684966, + "tokens_seen": 974979072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558676028084253, + "loss": 2.7667, + "theoretical_loss": 3.657997235293117, + "tokens_seen": 975044608 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035585757271815443, + "loss": 2.7782, + "theoretical_loss": 3.6579737285401825, + "tokens_seen": 975110144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035584754262788366, + "loss": 2.3444, + "theoretical_loss": 3.6579502238093835, + "tokens_seen": 975175680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558375125376129, + "loss": 2.7807, + "theoretical_loss": 3.6579267211004094, + "tokens_seen": 975241216 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355827482447342, + "loss": 2.7576, + "theoretical_loss": 3.657903220412951, + "tokens_seen": 975306752 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035581745235707126, + "loss": 2.8424, + "theoretical_loss": 3.657879721746699, + "tokens_seen": 975372288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558074222668004, + "loss": 2.7173, + "theoretical_loss": 3.657856225101343, + "tokens_seen": 975437824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557973921765296, + "loss": 2.7882, + "theoretical_loss": 3.6578327304765734, + "tokens_seen": 975503360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557873620862588, + "loss": 2.7136, + "theoretical_loss": 3.6578092378720815, + "tokens_seen": 975568896 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355777331995988, + "loss": 2.7584, + "theoretical_loss": 3.6577857472875572, + "tokens_seen": 975634432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035576730190571717, + "loss": 2.6165, + "theoretical_loss": 3.6577622587226917, + "tokens_seen": 975699968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035575727181544635, + "loss": 2.7483, + "theoretical_loss": 3.657738772177175, + "tokens_seen": 975765504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035574724172517553, + "loss": 2.5081, + "theoretical_loss": 3.6577152876506984, + "tokens_seen": 975831040 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035573721163490476, + "loss": 3.0255, + "theoretical_loss": 3.6576918051429526, + "tokens_seen": 975896576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557271815446339, + "loss": 2.4618, + "theoretical_loss": 3.6576683246536286, + "tokens_seen": 975962112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557171514543631, + "loss": 2.5939, + "theoretical_loss": 3.657644846182417, + "tokens_seen": 976027648 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035570712136409225, + "loss": 3.0158, + "theoretical_loss": 3.65762136972901, + "tokens_seen": 976093184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556970912738215, + "loss": 2.6443, + "theoretical_loss": 3.657597895293097, + "tokens_seen": 976158720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035568706118355067, + "loss": 2.648, + "theoretical_loss": 3.6575744228743705, + "tokens_seen": 976224256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035567703109327985, + "loss": 2.5602, + "theoretical_loss": 3.6575509524725214, + "tokens_seen": 976289792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035566700100300903, + "loss": 2.8696, + "theoretical_loss": 3.657527484087241, + "tokens_seen": 976355328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035565697091273827, + "loss": 2.7943, + "theoretical_loss": 3.6575040177182205, + "tokens_seen": 976420864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 546148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.82014799118042, + "objective/train/theoretical_loss": 3.6574805533651515, + "objective/train/tokens_used": 996946400, + "theoretical_loss": 3.6574805533651515, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556469408224674, + "loss": 2.6284, + "theoretical_loss": 3.6574805533651515, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035563691073219663, + "loss": 2.7933, + "theoretical_loss": 3.6574570910277258, + "tokens_seen": 976551936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035562688064192576, + "loss": 2.8423, + "theoretical_loss": 3.6574336307056345, + "tokens_seen": 976617472 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355616850551655, + "loss": 2.6661, + "theoretical_loss": 3.65741017239857, + "tokens_seen": 976683008 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035560682046138417, + "loss": 2.6578, + "theoretical_loss": 3.6573867161062235, + "tokens_seen": 976748544 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035559679037111335, + "loss": 2.6173, + "theoretical_loss": 3.6573632618282867, + "tokens_seen": 976814080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035558676028084253, + "loss": 2.6873, + "theoretical_loss": 3.6573398095644514, + "tokens_seen": 976879616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555767301905717, + "loss": 2.5844, + "theoretical_loss": 3.6573163593144105, + "tokens_seen": 976945152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555667001003009, + "loss": 2.7901, + "theoretical_loss": 3.657292911077855, + "tokens_seen": 977010688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035555667001003013, + "loss": 2.47, + "theoretical_loss": 3.657269464854477, + "tokens_seen": 977076224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035554663991975926, + "loss": 2.576, + "theoretical_loss": 3.657246020643969, + "tokens_seen": 977141760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555366098294885, + "loss": 2.7507, + "theoretical_loss": 3.6572225784460235, + "tokens_seen": 977207296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555265797392176, + "loss": 2.6684, + "theoretical_loss": 3.6571991382603324, + "tokens_seen": 977272832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035551654964894686, + "loss": 2.6614, + "theoretical_loss": 3.6571757000865874, + "tokens_seen": 977338368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035550651955867604, + "loss": 2.8629, + "theoretical_loss": 3.6571522639244822, + "tokens_seen": 977403904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554964894684052, + "loss": 2.6595, + "theoretical_loss": 3.6571288297737086, + "tokens_seen": 977469440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554864593781344, + "loss": 2.6743, + "theoretical_loss": 3.657105397633959, + "tokens_seen": 977534976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035547642928786364, + "loss": 2.7305, + "theoretical_loss": 3.657081967504926, + "tokens_seen": 977600512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035546639919759276, + "loss": 2.7035, + "theoretical_loss": 3.657058539386303, + "tokens_seen": 977666048 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355456369107322, + "loss": 2.7986, + "theoretical_loss": 3.6570351132777823, + "tokens_seen": 977731584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554463390170511, + "loss": 2.6578, + "theoretical_loss": 3.657011689179056, + "tokens_seen": 977797120 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035543630892678036, + "loss": 2.6389, + "theoretical_loss": 3.656988267089818, + "tokens_seen": 977862656 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035542627883650954, + "loss": 2.5991, + "theoretical_loss": 3.6569648470097613, + "tokens_seen": 977928192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554162487462387, + "loss": 2.7215, + "theoretical_loss": 3.6569414289385778, + "tokens_seen": 977993728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554062186559679, + "loss": 2.5016, + "theoretical_loss": 3.6569180128759617, + "tokens_seen": 978059264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 547377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.918895721435547, + "objective/train/theoretical_loss": 3.656894598821606, + "objective/train/tokens_used": 998584800, + "theoretical_loss": 3.656894598821606, + "tokens_seen": 978124800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553961885656971, + "loss": 2.7372, + "theoretical_loss": 3.656894598821606, + "tokens_seen": 978124800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035538615847542627, + "loss": 2.6627, + "theoretical_loss": 3.6568711867752035, + "tokens_seen": 978190336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553761283851555, + "loss": 2.7191, + "theoretical_loss": 3.6568477767364476, + "tokens_seen": 978255872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035536609829488463, + "loss": 2.7288, + "theoretical_loss": 3.656824368705032, + "tokens_seen": 978321408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035535606820461386, + "loss": 2.7022, + "theoretical_loss": 3.6568009626806495, + "tokens_seen": 978386944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035534603811434304, + "loss": 2.6384, + "theoretical_loss": 3.6567775586629945, + "tokens_seen": 978452480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553360080240722, + "loss": 2.8189, + "theoretical_loss": 3.6567541566517594, + "tokens_seen": 978518016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553259779338014, + "loss": 2.7914, + "theoretical_loss": 3.6567307566466387, + "tokens_seen": 978583552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553159478435306, + "loss": 2.8269, + "theoretical_loss": 3.656707358647326, + "tokens_seen": 978649088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035530591775325977, + "loss": 2.4179, + "theoretical_loss": 3.6566839626535153, + "tokens_seen": 978714624 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355295887662989, + "loss": 2.6729, + "theoretical_loss": 3.6566605686648996, + "tokens_seen": 978780160 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035528585757271813, + "loss": 2.6041, + "theoretical_loss": 3.6566371766811727, + "tokens_seen": 978845696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035527582748244737, + "loss": 2.5674, + "theoretical_loss": 3.65661378670203, + "tokens_seen": 978911232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552657973921765, + "loss": 2.6614, + "theoretical_loss": 3.6565903987271637, + "tokens_seen": 978976768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035525576730190573, + "loss": 2.8475, + "theoretical_loss": 3.6565670127562697, + "tokens_seen": 979042304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552457372116349, + "loss": 2.7853, + "theoretical_loss": 3.656543628789041, + "tokens_seen": 979107840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552357071213641, + "loss": 2.7585, + "theoretical_loss": 3.6565202468251723, + "tokens_seen": 979173376 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035522567703109327, + "loss": 2.7385, + "theoretical_loss": 3.6564968668643574, + "tokens_seen": 979238912 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035521564694082245, + "loss": 2.7629, + "theoretical_loss": 3.656473488906291, + "tokens_seen": 979304448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035520561685055163, + "loss": 2.501, + "theoretical_loss": 3.656450112950668, + "tokens_seen": 979369984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035519558676028087, + "loss": 2.6088, + "theoretical_loss": 3.6564267389971823, + "tokens_seen": 979435520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035518555667001, + "loss": 2.4397, + "theoretical_loss": 3.6564033670455283, + "tokens_seen": 979501056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035517552657973923, + "loss": 2.6795, + "theoretical_loss": 3.656379997095401, + "tokens_seen": 979566592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551654964894684, + "loss": 2.5704, + "theoretical_loss": 3.6563566291464955, + "tokens_seen": 979632128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551554663991976, + "loss": 2.7986, + "theoretical_loss": 3.6563332631985053, + "tokens_seen": 979697664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 547866, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.091559648513794, + "objective/train/theoretical_loss": 3.6563098992511267, + "objective/train/tokens_used": 1000223200, + "theoretical_loss": 3.6563098992511267, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551454363089268, + "loss": 2.9244, + "theoretical_loss": 3.6563098992511267, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035513540621865596, + "loss": 2.7605, + "theoretical_loss": 3.656286537304054, + "tokens_seen": 979828736 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035512537612838514, + "loss": 3.0127, + "theoretical_loss": 3.6562631773569816, + "tokens_seen": 979894272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035511534603811437, + "loss": 2.7358, + "theoretical_loss": 3.6562398194096053, + "tokens_seen": 979959808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551053159478435, + "loss": 2.5052, + "theoretical_loss": 3.65621646346162, + "tokens_seen": 980025344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035509528585757273, + "loss": 2.6234, + "theoretical_loss": 3.6561931095127207, + "tokens_seen": 980090880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550852557673019, + "loss": 2.7937, + "theoretical_loss": 3.656169757562603, + "tokens_seen": 980156416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550752256770311, + "loss": 2.7786, + "theoretical_loss": 3.6561464076109615, + "tokens_seen": 980221952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035506519558676033, + "loss": 2.5644, + "theoretical_loss": 3.6561230596574927, + "tokens_seen": 980287488 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035505516549648946, + "loss": 2.791, + "theoretical_loss": 3.656099713701891, + "tokens_seen": 980353024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550451354062187, + "loss": 2.7434, + "theoretical_loss": 3.6560763697438525, + "tokens_seen": 980418560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550351053159478, + "loss": 2.6858, + "theoretical_loss": 3.6560530277830723, + "tokens_seen": 980484096 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035502507522567706, + "loss": 2.6007, + "theoretical_loss": 3.656029687819246, + "tokens_seen": 980549632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035501504513540624, + "loss": 2.7632, + "theoretical_loss": 3.6560063498520705, + "tokens_seen": 980615168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550050150451354, + "loss": 2.6044, + "theoretical_loss": 3.65598301388124, + "tokens_seen": 980680704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549949849548646, + "loss": 2.7278, + "theoretical_loss": 3.6559596799064513, + "tokens_seen": 980746240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035498495486459384, + "loss": 2.6622, + "theoretical_loss": 3.6559363479274003, + "tokens_seen": 980811776 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035497492477432296, + "loss": 2.6397, + "theoretical_loss": 3.6559130179437824, + "tokens_seen": 980877312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549648946840522, + "loss": 2.4674, + "theoretical_loss": 3.655889689955294, + "tokens_seen": 980942848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549548645937813, + "loss": 2.5841, + "theoretical_loss": 3.6558663639616307, + "tokens_seen": 981008384 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035494483450351056, + "loss": 2.803, + "theoretical_loss": 3.6558430399624893, + "tokens_seen": 981073920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035493480441323974, + "loss": 2.8485, + "theoretical_loss": 3.655819717957566, + "tokens_seen": 981139456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549247743229689, + "loss": 2.7757, + "theoretical_loss": 3.655796397946557, + "tokens_seen": 981204992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549147442326981, + "loss": 2.6201, + "theoretical_loss": 3.6557730799291583, + "tokens_seen": 981270528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549047141424273, + "loss": 2.6034, + "theoretical_loss": 3.655749763905067, + "tokens_seen": 981336064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 549153, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9054205417633057, + "objective/train/theoretical_loss": 3.655726449873979, + "objective/train/tokens_used": 1001861600, + "theoretical_loss": 3.655726449873979, + "tokens_seen": 981401600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035489468405215647, + "loss": 2.9853, + "theoretical_loss": 3.655726449873979, + "tokens_seen": 981401600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548846539618857, + "loss": 2.9481, + "theoretical_loss": 3.6557031378355913, + "tokens_seen": 981467136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035487462387161483, + "loss": 2.6993, + "theoretical_loss": 3.6556798277896005, + "tokens_seen": 981532672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035486459378134406, + "loss": 2.8664, + "theoretical_loss": 3.655656519735703, + "tokens_seen": 981598208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035485456369107324, + "loss": 2.7443, + "theoretical_loss": 3.655633213673596, + "tokens_seen": 981663744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548445336008024, + "loss": 2.4849, + "theoretical_loss": 3.6556099096029757, + "tokens_seen": 981729280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548345035105316, + "loss": 2.8908, + "theoretical_loss": 3.6555866075235395, + "tokens_seen": 981794816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548244734202608, + "loss": 2.4488, + "theoretical_loss": 3.655563307434984, + "tokens_seen": 981860352 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035481444332998997, + "loss": 2.7768, + "theoretical_loss": 3.655540009337007, + "tokens_seen": 981925888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548044132397192, + "loss": 2.7123, + "theoretical_loss": 3.6555167132293045, + "tokens_seen": 981991424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035479438314944833, + "loss": 2.587, + "theoretical_loss": 3.6554934191115747, + "tokens_seen": 982056960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035478435305917757, + "loss": 2.6422, + "theoretical_loss": 3.6554701269835146, + "tokens_seen": 982122496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547743229689067, + "loss": 2.6564, + "theoretical_loss": 3.6554468368448205, + "tokens_seen": 982188032 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035476429287863593, + "loss": 2.6752, + "theoretical_loss": 3.6554235486951914, + "tokens_seen": 982253568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547542627883651, + "loss": 2.6921, + "theoretical_loss": 3.6554002625343234, + "tokens_seen": 982319104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547442326980943, + "loss": 2.7904, + "theoretical_loss": 3.6553769783619146, + "tokens_seen": 982384640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035473420260782347, + "loss": 2.7994, + "theoretical_loss": 3.6553536961776625, + "tokens_seen": 982450176 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035472417251755265, + "loss": 2.6377, + "theoretical_loss": 3.6553304159812647, + "tokens_seen": 982515712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035471414242728183, + "loss": 2.8887, + "theoretical_loss": 3.655307137772419, + "tokens_seen": 982581248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035470411233701107, + "loss": 2.7482, + "theoretical_loss": 3.655283861550823, + "tokens_seen": 982646784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546940822467402, + "loss": 2.5649, + "theoretical_loss": 3.6552605873161745, + "tokens_seen": 982712320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035468405215646943, + "loss": 2.7879, + "theoretical_loss": 3.655237315068171, + "tokens_seen": 982777856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546740220661986, + "loss": 2.8483, + "theoretical_loss": 3.655214044806512, + "tokens_seen": 982843392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546639919759278, + "loss": 2.7856, + "theoretical_loss": 3.6551907765308935, + "tokens_seen": 982908928 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354653961885657, + "loss": 2.7548, + "theoretical_loss": 3.655167510241015, + "tokens_seen": 982974464 + }, + { + "debugging/Self-BLEU-5": 0.5000497629393682, + "debugging/distinct-1-grams": 0.7839180687368331, + "debugging/distinct-2-grams": 0.9723027630693535, + "debugging/entropy-1-grams": 5.462779182741081, + "debugging/entropy-2-grams": 6.200783858933273, + "debugging/length": 666.1666666666666, + "debugging/num_segments": 6, + "epoch": 0.3, + "objective/train/docs_used": 549695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.27803897857666, + "objective/train/theoretical_loss": 3.655144245936574, + "objective/train/tokens_used": 1003500000, + "theoretical_loss": 3.655144245936574, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035464393179538616, + "loss": 2.4059, + "theoretical_loss": 3.655144245936574, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035463390170511534, + "loss": 2.8016, + "theoretical_loss": 3.6551209836172696, + "tokens_seen": 983105536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035462387161484457, + "loss": 2.5838, + "theoretical_loss": 3.6550977232827986, + "tokens_seen": 983171072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546138415245737, + "loss": 2.5899, + "theoretical_loss": 3.65507446493286, + "tokens_seen": 983236608 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035460381143430293, + "loss": 2.74, + "theoretical_loss": 3.655051208567153, + "tokens_seen": 983302144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035459378134403206, + "loss": 2.8234, + "theoretical_loss": 3.655027954185376, + "tokens_seen": 983367680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545837512537613, + "loss": 2.7215, + "theoretical_loss": 3.6550047017872265, + "tokens_seen": 983433216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545737211634905, + "loss": 2.6825, + "theoretical_loss": 3.6549814513724037, + "tokens_seen": 983498752 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035456369107321966, + "loss": 2.6115, + "theoretical_loss": 3.6549582029406062, + "tokens_seen": 983564288 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035455366098294884, + "loss": 2.7599, + "theoretical_loss": 3.654934956491533, + "tokens_seen": 983629824 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354543630892678, + "loss": 2.7921, + "theoretical_loss": 3.654911712024883, + "tokens_seen": 983695360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545336008024072, + "loss": 2.7611, + "theoretical_loss": 3.6548884695403543, + "tokens_seen": 983760896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035452357071213644, + "loss": 2.7598, + "theoretical_loss": 3.6548652290376467, + "tokens_seen": 983826432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035451354062186556, + "loss": 2.7005, + "theoretical_loss": 3.654841990516459, + "tokens_seen": 983891968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545035105315948, + "loss": 2.5206, + "theoretical_loss": 3.65481875397649, + "tokens_seen": 983957504 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354493480441324, + "loss": 2.5426, + "theoretical_loss": 3.654795519417439, + "tokens_seen": 984023040 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035448345035105316, + "loss": 2.8029, + "theoretical_loss": 3.6547722868390053, + "tokens_seen": 984088576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035447342026078234, + "loss": 2.813, + "theoretical_loss": 3.654749056240888, + "tokens_seen": 984154112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544633901705115, + "loss": 2.5548, + "theoretical_loss": 3.6547258276227863, + "tokens_seen": 984219648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544533600802407, + "loss": 2.6819, + "theoretical_loss": 3.6547026009844004, + "tokens_seen": 984285184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035444332998996994, + "loss": 2.5199, + "theoretical_loss": 3.6546793763254293, + "tokens_seen": 984350720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035443329989969907, + "loss": 2.7544, + "theoretical_loss": 3.6546561536455724, + "tokens_seen": 984416256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544232698094283, + "loss": 2.6461, + "theoretical_loss": 3.654632932944529, + "tokens_seen": 984481792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035441323971915743, + "loss": 2.6753, + "theoretical_loss": 3.654609714222, + "tokens_seen": 984547328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035440320962888667, + "loss": 2.6239, + "theoretical_loss": 3.654586497477684, + "tokens_seen": 984612864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 550994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.944199562072754, + "objective/train/theoretical_loss": 3.6545632827112806, + "objective/train/tokens_used": 1005138400, + "theoretical_loss": 3.6545632827112806, + "tokens_seen": 984678400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035439317953861585, + "loss": 2.7087, + "theoretical_loss": 3.6545632827112806, + "tokens_seen": 984678400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035438314944834503, + "loss": 2.7407, + "theoretical_loss": 3.654540069922491, + "tokens_seen": 984743936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543731193580742, + "loss": 2.7961, + "theoretical_loss": 3.6545168591110135, + "tokens_seen": 984809472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035436308926780344, + "loss": 2.5277, + "theoretical_loss": 3.6544936502765495, + "tokens_seen": 984875008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543530591775326, + "loss": 2.5603, + "theoretical_loss": 3.6544704434187985, + "tokens_seen": 984940544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543430290872618, + "loss": 2.4406, + "theoretical_loss": 3.6544472385374602, + "tokens_seen": 985006080 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354332998996991, + "loss": 2.9052, + "theoretical_loss": 3.654424035632236, + "tokens_seen": 985071616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035432296890672017, + "loss": 2.5012, + "theoretical_loss": 3.654400834702825, + "tokens_seen": 985137152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543129388164494, + "loss": 2.8039, + "theoretical_loss": 3.6543776357489275, + "tokens_seen": 985202688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035430290872617853, + "loss": 2.7412, + "theoretical_loss": 3.6543544387702447, + "tokens_seen": 985268224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035429287863590777, + "loss": 2.9997, + "theoretical_loss": 3.654331243766477, + "tokens_seen": 985333760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542828485456369, + "loss": 2.8078, + "theoretical_loss": 3.654308050737324, + "tokens_seen": 985399296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035427281845536613, + "loss": 2.6121, + "theoretical_loss": 3.6542848596824875, + "tokens_seen": 985464832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542627883650953, + "loss": 2.5112, + "theoretical_loss": 3.6542616706016675, + "tokens_seen": 985530368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542527582748245, + "loss": 2.4367, + "theoretical_loss": 3.6542384834945643, + "tokens_seen": 985595904 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035424272818455367, + "loss": 2.6558, + "theoretical_loss": 3.654215298360879, + "tokens_seen": 985661440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035423269809428285, + "loss": 2.5777, + "theoretical_loss": 3.6541921152003134, + "tokens_seen": 985726976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035422266800401203, + "loss": 2.7154, + "theoretical_loss": 3.654168934012567, + "tokens_seen": 985792512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035421263791374127, + "loss": 2.471, + "theoretical_loss": 3.6541457547973417, + "tokens_seen": 985858048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542026078234704, + "loss": 2.8969, + "theoretical_loss": 3.654122577554338, + "tokens_seen": 985923584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035419257773319963, + "loss": 2.5885, + "theoretical_loss": 3.6540994022832574, + "tokens_seen": 985989120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541825476429288, + "loss": 2.7422, + "theoretical_loss": 3.6540762289838007, + "tokens_seen": 986054656 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354172517552658, + "loss": 2.8467, + "theoretical_loss": 3.65405305765567, + "tokens_seen": 986120192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541624874623872, + "loss": 2.7443, + "theoretical_loss": 3.654029888298566, + "tokens_seen": 986185728 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035415245737211636, + "loss": 2.5481, + "theoretical_loss": 3.6540067209121894, + "tokens_seen": 986251264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 551567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.91987681388855, + "objective/train/theoretical_loss": 3.653983555496242, + "objective/train/tokens_used": 1006776800, + "theoretical_loss": 3.653983555496242, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035414242728184554, + "loss": 2.6924, + "theoretical_loss": 3.653983555496242, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035413239719157477, + "loss": 2.7576, + "theoretical_loss": 3.6539603920504264, + "tokens_seen": 986382336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541223671013039, + "loss": 2.6437, + "theoretical_loss": 3.653937230574443, + "tokens_seen": 986447872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035411233701103314, + "loss": 2.4609, + "theoretical_loss": 3.653914071067994, + "tokens_seen": 986513408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035410230692076226, + "loss": 2.7274, + "theoretical_loss": 3.6538909135307813, + "tokens_seen": 986578944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540922768304915, + "loss": 2.7185, + "theoretical_loss": 3.653867757962505, + "tokens_seen": 986644480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540822467402207, + "loss": 2.4144, + "theoretical_loss": 3.6538446043628694, + "tokens_seen": 986710016 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035407221664994986, + "loss": 2.9986, + "theoretical_loss": 3.6538214527315747, + "tokens_seen": 986775552 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035406218655967904, + "loss": 2.7329, + "theoretical_loss": 3.6537983030683234, + "tokens_seen": 986841088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540521564694082, + "loss": 2.5728, + "theoretical_loss": 3.6537751553728173, + "tokens_seen": 986906624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540421263791374, + "loss": 2.6988, + "theoretical_loss": 3.6537520096447587, + "tokens_seen": 986972160 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035403209628886664, + "loss": 2.8755, + "theoretical_loss": 3.65372886588385, + "tokens_seen": 987037696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035402206619859577, + "loss": 2.4468, + "theoretical_loss": 3.653705724089793, + "tokens_seen": 987103232 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354012036108325, + "loss": 2.6212, + "theoretical_loss": 3.65368258426229, + "tokens_seen": 987168768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540020060180542, + "loss": 2.5444, + "theoretical_loss": 3.6536594464010435, + "tokens_seen": 987234304 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035399197592778336, + "loss": 2.7203, + "theoretical_loss": 3.653636310505756, + "tokens_seen": 987299840 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035398194583751254, + "loss": 2.7516, + "theoretical_loss": 3.6536131765761297, + "tokens_seen": 987365376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539719157472417, + "loss": 2.6541, + "theoretical_loss": 3.653590044611867, + "tokens_seen": 987430912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539618856569709, + "loss": 2.6099, + "theoretical_loss": 3.653566914612671, + "tokens_seen": 987496448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035395185556670014, + "loss": 2.7547, + "theoretical_loss": 3.653543786578244, + "tokens_seen": 987561984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035394182547642927, + "loss": 2.8886, + "theoretical_loss": 3.6535206605082893, + "tokens_seen": 987627520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539317953861585, + "loss": 2.7337, + "theoretical_loss": 3.653497536402509, + "tokens_seen": 987693056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035392176529588763, + "loss": 2.7313, + "theoretical_loss": 3.6534744142606064, + "tokens_seen": 987758592 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035391173520561687, + "loss": 2.7238, + "theoretical_loss": 3.653451294082284, + "tokens_seen": 987824128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035390170511534605, + "loss": 2.7589, + "theoretical_loss": 3.653428175867245, + "tokens_seen": 987889664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 552650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.231137752532959, + "objective/train/theoretical_loss": 3.653405059615193, + "objective/train/tokens_used": 1008415200, + "theoretical_loss": 3.653405059615193, + "tokens_seen": 987955200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035389167502507523, + "loss": 2.8518, + "theoretical_loss": 3.653405059615193, + "tokens_seen": 987955200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538816449348044, + "loss": 2.7992, + "theoretical_loss": 3.65338194532583, + "tokens_seen": 988020736 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035387161484453364, + "loss": 2.4641, + "theoretical_loss": 3.6533588329988604, + "tokens_seen": 988086272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035386158475426277, + "loss": 2.7381, + "theoretical_loss": 3.6533357226339866, + "tokens_seen": 988151808 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353851554663992, + "loss": 2.7929, + "theoretical_loss": 3.653312614230912, + "tokens_seen": 988217344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035384152457372113, + "loss": 2.8832, + "theoretical_loss": 3.653289507789341, + "tokens_seen": 988282880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035383149448345037, + "loss": 2.6008, + "theoretical_loss": 3.653266403308976, + "tokens_seen": 988348416 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035382146439317955, + "loss": 2.7089, + "theoretical_loss": 3.65324330078952, + "tokens_seen": 988413952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035381143430290873, + "loss": 2.5198, + "theoretical_loss": 3.6532202002306784, + "tokens_seen": 988479488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538014042126379, + "loss": 2.7445, + "theoretical_loss": 3.653197101632153, + "tokens_seen": 988545024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537913741223671, + "loss": 2.627, + "theoretical_loss": 3.653174004993648, + "tokens_seen": 988610560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537813440320963, + "loss": 2.7365, + "theoretical_loss": 3.653150910314868, + "tokens_seen": 988676096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537713139418255, + "loss": 2.6624, + "theoretical_loss": 3.653127817595516, + "tokens_seen": 988741632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035376128385155464, + "loss": 2.8601, + "theoretical_loss": 3.6531047268352967, + "tokens_seen": 988807168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035375125376128387, + "loss": 2.7081, + "theoretical_loss": 3.653081638033913, + "tokens_seen": 988872704 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353741223671013, + "loss": 2.7607, + "theoretical_loss": 3.65305855119107, + "tokens_seen": 988938240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035373119358074223, + "loss": 2.6208, + "theoretical_loss": 3.6530354663064704, + "tokens_seen": 989003776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537211634904714, + "loss": 2.6153, + "theoretical_loss": 3.65301238337982, + "tokens_seen": 989069312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537111334002006, + "loss": 2.839, + "theoretical_loss": 3.6529893024108215, + "tokens_seen": 989134848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537011033099298, + "loss": 2.5253, + "theoretical_loss": 3.65296622339918, + "tokens_seen": 989200384 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353691073219659, + "loss": 2.7016, + "theoretical_loss": 3.6529431463446, + "tokens_seen": 989265920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035368104312938814, + "loss": 2.6507, + "theoretical_loss": 3.652920071246786, + "tokens_seen": 989331456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536710130391174, + "loss": 2.4649, + "theoretical_loss": 3.652896998105442, + "tokens_seen": 989396992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536609829488465, + "loss": 2.844, + "theoretical_loss": 3.652873926920272, + "tokens_seen": 989462528 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035365095285857574, + "loss": 2.8394, + "theoretical_loss": 3.652850857690982, + "tokens_seen": 989528064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 553159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4328830242156982, + "objective/train/theoretical_loss": 3.6528277904172755, + "objective/train/tokens_used": 1010053600, + "theoretical_loss": 3.6528277904172755, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536409227683049, + "loss": 2.5894, + "theoretical_loss": 3.6528277904172755, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536308926780341, + "loss": 2.3536, + "theoretical_loss": 3.652804725098858, + "tokens_seen": 989659136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536208625877633, + "loss": 2.737, + "theoretical_loss": 3.6527816617354336, + "tokens_seen": 989724672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035361083249749246, + "loss": 2.4307, + "theoretical_loss": 3.652758600326708, + "tokens_seen": 989790208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536008024072217, + "loss": 2.6266, + "theoretical_loss": 3.652735540872385, + "tokens_seen": 989855744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535907723169509, + "loss": 2.5696, + "theoretical_loss": 3.652712483372171, + "tokens_seen": 989921280 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035358074222668006, + "loss": 2.7762, + "theoretical_loss": 3.65268942782577, + "tokens_seen": 989986816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035357071213640924, + "loss": 2.707, + "theoretical_loss": 3.652666374232887, + "tokens_seen": 990052352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535606820461384, + "loss": 2.5712, + "theoretical_loss": 3.652643322593228, + "tokens_seen": 990117888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535506519558676, + "loss": 2.6914, + "theoretical_loss": 3.652620272906498, + "tokens_seen": 990183424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035354062186559684, + "loss": 2.5723, + "theoretical_loss": 3.6525972251724017, + "tokens_seen": 990248960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035353059177532597, + "loss": 2.4504, + "theoretical_loss": 3.652574179390645, + "tokens_seen": 990314496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535205616850552, + "loss": 2.5014, + "theoretical_loss": 3.6525511355609335, + "tokens_seen": 990380032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535105315947844, + "loss": 2.7155, + "theoretical_loss": 3.6525280936829723, + "tokens_seen": 990445568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035350050150451356, + "loss": 2.6203, + "theoretical_loss": 3.6525050537564674, + "tokens_seen": 990511104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035349047141424274, + "loss": 2.4302, + "theoretical_loss": 3.652482015781124, + "tokens_seen": 990576640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534804413239719, + "loss": 2.7063, + "theoretical_loss": 3.6524589797566476, + "tokens_seen": 990642176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534704112337011, + "loss": 2.6519, + "theoretical_loss": 3.6524359456827447, + "tokens_seen": 990707712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035346038114343034, + "loss": 2.5755, + "theoretical_loss": 3.6524129135591203, + "tokens_seen": 990773248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035345035105315947, + "loss": 2.6261, + "theoretical_loss": 3.652389883385481, + "tokens_seen": 990838784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534403209628887, + "loss": 2.5731, + "theoretical_loss": 3.6523668551615325, + "tokens_seen": 990904320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035343029087261783, + "loss": 2.7021, + "theoretical_loss": 3.6523438288869805, + "tokens_seen": 990969856 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035342026078234707, + "loss": 2.7016, + "theoretical_loss": 3.6523208045615316, + "tokens_seen": 991035392 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035341023069207625, + "loss": 2.5318, + "theoretical_loss": 3.652297782184892, + "tokens_seen": 991100928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035340020060180543, + "loss": 2.6415, + "theoretical_loss": 3.6522747617567672, + "tokens_seen": 991166464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 554508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6566503047943115, + "objective/train/theoretical_loss": 3.6522517432768637, + "objective/train/tokens_used": 1011692000, + "theoretical_loss": 3.6522517432768637, + "tokens_seen": 991232000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533901705115346, + "loss": 2.6658, + "theoretical_loss": 3.6522517432768637, + "tokens_seen": 991232000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035338014042126384, + "loss": 2.6459, + "theoretical_loss": 3.6522287267448883, + "tokens_seen": 991297536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035337011033099297, + "loss": 2.5836, + "theoretical_loss": 3.652205712160547, + "tokens_seen": 991363072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533600802407222, + "loss": 2.8489, + "theoretical_loss": 3.652182699523546, + "tokens_seen": 991428608 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035335005015045133, + "loss": 2.7108, + "theoretical_loss": 3.6521596888335925, + "tokens_seen": 991494144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035334002006018057, + "loss": 2.5673, + "theoretical_loss": 3.6521366800903925, + "tokens_seen": 991559680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035332998996990975, + "loss": 2.727, + "theoretical_loss": 3.6521136732936528, + "tokens_seen": 991625216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035331995987963893, + "loss": 2.7111, + "theoretical_loss": 3.6520906684430807, + "tokens_seen": 991690752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533099297893681, + "loss": 2.5756, + "theoretical_loss": 3.652067665538382, + "tokens_seen": 991756288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532998996990973, + "loss": 2.5974, + "theoretical_loss": 3.652044664579264, + "tokens_seen": 991821824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532898696088265, + "loss": 2.425, + "theoretical_loss": 3.652021665565434, + "tokens_seen": 991887360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532798395185557, + "loss": 2.6744, + "theoretical_loss": 3.6519986684965984, + "tokens_seen": 991952896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035326980942828484, + "loss": 2.7236, + "theoretical_loss": 3.6519756733724646, + "tokens_seen": 992018432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035325977933801407, + "loss": 2.6378, + "theoretical_loss": 3.651952680192739, + "tokens_seen": 992083968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532497492477432, + "loss": 2.6454, + "theoretical_loss": 3.6519296889571295, + "tokens_seen": 992149504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035323971915747243, + "loss": 2.6259, + "theoretical_loss": 3.651906699665343, + "tokens_seen": 992215040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532296890672016, + "loss": 2.5189, + "theoretical_loss": 3.651883712317087, + "tokens_seen": 992280576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532196589769308, + "loss": 2.8199, + "theoretical_loss": 3.6518607269120684, + "tokens_seen": 992346112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035320962888666, + "loss": 2.6205, + "theoretical_loss": 3.6518377434499953, + "tokens_seen": 992411648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531995987963892, + "loss": 2.8705, + "theoretical_loss": 3.651814761930575, + "tokens_seen": 992477184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035318956870611834, + "loss": 2.7282, + "theoretical_loss": 3.651791782353514, + "tokens_seen": 992542720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531795386158476, + "loss": 2.5389, + "theoretical_loss": 3.6517688047185217, + "tokens_seen": 992608256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531695085255767, + "loss": 2.6315, + "theoretical_loss": 3.6517458290253044, + "tokens_seen": 992673792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035315947843530594, + "loss": 2.7756, + "theoretical_loss": 3.6517228552735697, + "tokens_seen": 992739328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531494483450351, + "loss": 2.6943, + "theoretical_loss": 3.651699883463026, + "tokens_seen": 992804864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 555131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2285315990448, + "objective/train/theoretical_loss": 3.6516769135933815, + "objective/train/tokens_used": 1013330400, + "theoretical_loss": 3.6516769135933815, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531394182547643, + "loss": 2.5577, + "theoretical_loss": 3.6516769135933815, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531293881644935, + "loss": 2.6386, + "theoretical_loss": 3.6516539456643438, + "tokens_seen": 992935936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035311935807422266, + "loss": 2.8148, + "theoretical_loss": 3.6516309796756206, + "tokens_seen": 993001472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035310932798395184, + "loss": 2.7412, + "theoretical_loss": 3.65160801562692, + "tokens_seen": 993067008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530992978936811, + "loss": 2.8597, + "theoretical_loss": 3.6515850535179504, + "tokens_seen": 993132544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530892678034102, + "loss": 2.4166, + "theoretical_loss": 3.65156209334842, + "tokens_seen": 993198080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035307923771313944, + "loss": 2.5507, + "theoretical_loss": 3.651539135118036, + "tokens_seen": 993263616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035306920762286857, + "loss": 2.7169, + "theoretical_loss": 3.6515161788265083, + "tokens_seen": 993329152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530591775325978, + "loss": 2.607, + "theoretical_loss": 3.6514932244735436, + "tokens_seen": 993394688 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353049147442327, + "loss": 2.7023, + "theoretical_loss": 3.6514702720588517, + "tokens_seen": 993460224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035303911735205617, + "loss": 2.3688, + "theoretical_loss": 3.6514473215821406, + "tokens_seen": 993525760 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035302908726178535, + "loss": 2.7075, + "theoretical_loss": 3.6514243730431186, + "tokens_seen": 993591296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530190571715146, + "loss": 2.4497, + "theoretical_loss": 3.6514014264414953, + "tokens_seen": 993656832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530090270812437, + "loss": 2.8638, + "theoretical_loss": 3.6513784817769777, + "tokens_seen": 993722368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035299899699097294, + "loss": 2.8829, + "theoretical_loss": 3.6513555390492756, + "tokens_seen": 993787904 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035298896690070207, + "loss": 2.6749, + "theoretical_loss": 3.6513325982580978, + "tokens_seen": 993853440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529789368104313, + "loss": 2.4084, + "theoretical_loss": 3.651309659403153, + "tokens_seen": 993918976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529689067201605, + "loss": 2.6559, + "theoretical_loss": 3.6512867224841505, + "tokens_seen": 993984512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035295887662988967, + "loss": 2.6972, + "theoretical_loss": 3.6512637875007985, + "tokens_seen": 994050048 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035294884653961885, + "loss": 2.8845, + "theoretical_loss": 3.6512408544528063, + "tokens_seen": 994115584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035293881644934803, + "loss": 2.7391, + "theoretical_loss": 3.6512179233398836, + "tokens_seen": 994181120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529287863590772, + "loss": 2.8006, + "theoretical_loss": 3.651194994161739, + "tokens_seen": 994246656 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035291875626880645, + "loss": 2.6687, + "theoretical_loss": 3.6511720669180816, + "tokens_seen": 994312192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529087261785356, + "loss": 2.5931, + "theoretical_loss": 3.6511491416086215, + "tokens_seen": 994377728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528986960882648, + "loss": 2.7499, + "theoretical_loss": 3.6511262182330673, + "tokens_seen": 994443264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 556491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.311375617980957, + "objective/train/theoretical_loss": 3.6511032967911285, + "objective/train/tokens_used": 1014968800, + "theoretical_loss": 3.6511032967911285, + "tokens_seen": 994508800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035288866599799394, + "loss": 2.5432, + "theoretical_loss": 3.6511032967911285, + "tokens_seen": 994508800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035287863590772317, + "loss": 2.6023, + "theoretical_loss": 3.6510803772825153, + "tokens_seen": 994574336 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035286860581745235, + "loss": 2.411, + "theoretical_loss": 3.6510574597069367, + "tokens_seen": 994639872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035285857572718153, + "loss": 2.6745, + "theoretical_loss": 3.6510345440641023, + "tokens_seen": 994705408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035284854563691077, + "loss": 2.8324, + "theoretical_loss": 3.6510116303537217, + "tokens_seen": 994770944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035283851554663995, + "loss": 2.6422, + "theoretical_loss": 3.6509887185755048, + "tokens_seen": 994836480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035282848545636913, + "loss": 2.4072, + "theoretical_loss": 3.650965808729161, + "tokens_seen": 994902016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528184553660983, + "loss": 2.4776, + "theoretical_loss": 3.6509429008144014, + "tokens_seen": 994967552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528084252758275, + "loss": 2.7632, + "theoretical_loss": 3.650919994830935, + "tokens_seen": 995033088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527983951855567, + "loss": 2.588, + "theoretical_loss": 3.6508970907784715, + "tokens_seen": 995098624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527883650952859, + "loss": 2.5729, + "theoretical_loss": 3.6508741886567213, + "tokens_seen": 995164160 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035277833500501504, + "loss": 2.7314, + "theoretical_loss": 3.650851288465395, + "tokens_seen": 995229696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035276830491474427, + "loss": 2.7634, + "theoretical_loss": 3.650828390204202, + "tokens_seen": 995295232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527582748244734, + "loss": 2.6234, + "theoretical_loss": 3.650805493872853, + "tokens_seen": 995360768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035274824473420263, + "loss": 2.8951, + "theoretical_loss": 3.650782599471058, + "tokens_seen": 995426304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527382146439318, + "loss": 2.6232, + "theoretical_loss": 3.650759706998528, + "tokens_seen": 995491840 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352728184553661, + "loss": 2.6352, + "theoretical_loss": 3.6507368164549727, + "tokens_seen": 995557376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527181544633902, + "loss": 2.9256, + "theoretical_loss": 3.650713927840103, + "tokens_seen": 995622912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527081243731194, + "loss": 2.4447, + "theoretical_loss": 3.650691041153629, + "tokens_seen": 995688448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035269809428284854, + "loss": 2.6017, + "theoretical_loss": 3.6506681563952625, + "tokens_seen": 995753984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526880641925778, + "loss": 2.6119, + "theoretical_loss": 3.6506452735647126, + "tokens_seen": 995819520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526780341023069, + "loss": 2.8533, + "theoretical_loss": 3.650622392661691, + "tokens_seen": 995885056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035266800401203614, + "loss": 2.7495, + "theoretical_loss": 3.6505995136859086, + "tokens_seen": 995950592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526579739217653, + "loss": 2.5176, + "theoretical_loss": 3.6505766366370755, + "tokens_seen": 996016128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526479438314945, + "loss": 2.5834, + "theoretical_loss": 3.6505537615149035, + "tokens_seen": 996081664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 557069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.978800654411316, + "objective/train/theoretical_loss": 3.650530888319103, + "objective/train/tokens_used": 1016607200, + "theoretical_loss": 3.650530888319103, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526379137412237, + "loss": 2.3804, + "theoretical_loss": 3.650530888319103, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035262788365095286, + "loss": 2.7149, + "theoretical_loss": 3.6505080170493853, + "tokens_seen": 996212736 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035261785356068204, + "loss": 2.6825, + "theoretical_loss": 3.6504851477054614, + "tokens_seen": 996278272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526078234704113, + "loss": 2.9308, + "theoretical_loss": 3.6504622802870426, + "tokens_seen": 996343808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525977933801404, + "loss": 2.7761, + "theoretical_loss": 3.6504394147938406, + "tokens_seen": 996409344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035258776328986964, + "loss": 2.7075, + "theoretical_loss": 3.6504165512255655, + "tokens_seen": 996474880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035257773319959877, + "loss": 2.6206, + "theoretical_loss": 3.6503936895819296, + "tokens_seen": 996540416 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352567703109328, + "loss": 2.5507, + "theoretical_loss": 3.650370829862644, + "tokens_seen": 996605952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525576730190572, + "loss": 2.5328, + "theoretical_loss": 3.6503479720674203, + "tokens_seen": 996671488 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035254764292878637, + "loss": 2.7378, + "theoretical_loss": 3.6503251161959707, + "tokens_seen": 996737024 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035253761283851555, + "loss": 2.6422, + "theoretical_loss": 3.6503022622480055, + "tokens_seen": 996802560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525275827482448, + "loss": 2.5801, + "theoretical_loss": 3.650279410223237, + "tokens_seen": 996868096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525175526579739, + "loss": 2.6606, + "theoretical_loss": 3.6502565601213774, + "tokens_seen": 996933632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035250752256770314, + "loss": 2.4713, + "theoretical_loss": 3.6502337119421377, + "tokens_seen": 996999168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035249749247743227, + "loss": 2.68, + "theoretical_loss": 3.6502108656852306, + "tokens_seen": 997064704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524874623871615, + "loss": 2.5103, + "theoretical_loss": 3.6501880213503672, + "tokens_seen": 997130240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524774322968907, + "loss": 2.6101, + "theoretical_loss": 3.65016517893726, + "tokens_seen": 997195776 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035246740220661987, + "loss": 2.6795, + "theoretical_loss": 3.650142338445621, + "tokens_seen": 997261312 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035245737211634905, + "loss": 2.6608, + "theoretical_loss": 3.6501194998751623, + "tokens_seen": 997326848 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035244734202607823, + "loss": 2.7978, + "theoretical_loss": 3.650096663225596, + "tokens_seen": 997392384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524373119358074, + "loss": 2.7366, + "theoretical_loss": 3.650073828496634, + "tokens_seen": 997457920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035242728184553665, + "loss": 2.9977, + "theoretical_loss": 3.650050995687989, + "tokens_seen": 997523456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524172517552658, + "loss": 2.9039, + "theoretical_loss": 3.650028164799374, + "tokens_seen": 997588992 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352407221664995, + "loss": 2.6053, + "theoretical_loss": 3.6500053358305005, + "tokens_seen": 997654528 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035239719157472414, + "loss": 2.6113, + "theoretical_loss": 3.649982508781081, + "tokens_seen": 997720064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 558090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5171451568603516, + "objective/train/theoretical_loss": 3.6499596836508283, + "objective/train/tokens_used": 1018245600, + "theoretical_loss": 3.6499596836508283, + "tokens_seen": 997785600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035238716148445337, + "loss": 2.7921, + "theoretical_loss": 3.6499596836508283, + "tokens_seen": 997785600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035237713139418255, + "loss": 2.5081, + "theoretical_loss": 3.649936860439455, + "tokens_seen": 997851136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035236710130391173, + "loss": 2.8577, + "theoretical_loss": 3.6499140391466733, + "tokens_seen": 997916672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523570712136409, + "loss": 2.6762, + "theoretical_loss": 3.649891219772197, + "tokens_seen": 997982208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035234704112337015, + "loss": 2.6309, + "theoretical_loss": 3.6498684023157386, + "tokens_seen": 998047744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523370110330993, + "loss": 2.9457, + "theoretical_loss": 3.64984558677701, + "tokens_seen": 998113280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523269809428285, + "loss": 2.7832, + "theoretical_loss": 3.6498227731557247, + "tokens_seen": 998178816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035231695085255764, + "loss": 2.6664, + "theoretical_loss": 3.649799961451596, + "tokens_seen": 998244352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523069207622869, + "loss": 2.4468, + "theoretical_loss": 3.6497771516643365, + "tokens_seen": 998309888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035229689067201606, + "loss": 2.6363, + "theoretical_loss": 3.64975434379366, + "tokens_seen": 998375424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035228686058174524, + "loss": 2.6484, + "theoretical_loss": 3.649731537839279, + "tokens_seen": 998440960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522768304914744, + "loss": 2.6946, + "theoretical_loss": 3.6497087338009067, + "tokens_seen": 998506496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522668004012036, + "loss": 2.8492, + "theoretical_loss": 3.649685931678256, + "tokens_seen": 998572032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522567703109328, + "loss": 2.3118, + "theoretical_loss": 3.6496631314710415, + "tokens_seen": 998637568 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352246740220662, + "loss": 2.8642, + "theoretical_loss": 3.6496403331789757, + "tokens_seen": 998703104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035223671013039114, + "loss": 2.4645, + "theoretical_loss": 3.649617536801773, + "tokens_seen": 998768640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522266800401204, + "loss": 2.5693, + "theoretical_loss": 3.6495947423391453, + "tokens_seen": 998834176 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035221664994984956, + "loss": 2.7414, + "theoretical_loss": 3.6495719497908077, + "tokens_seen": 998899712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035220661985957874, + "loss": 2.7699, + "theoretical_loss": 3.6495491591564733, + "tokens_seen": 998965248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521965897693079, + "loss": 2.698, + "theoretical_loss": 3.6495263704358556, + "tokens_seen": 999030784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521865596790371, + "loss": 2.4147, + "theoretical_loss": 3.6495035836286687, + "tokens_seen": 999096320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521765295887663, + "loss": 2.511, + "theoretical_loss": 3.6494807987346265, + "tokens_seen": 999161856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521664994984955, + "loss": 2.9637, + "theoretical_loss": 3.649458015753442, + "tokens_seen": 999227392 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035215646940822465, + "loss": 2.5515, + "theoretical_loss": 3.649435234684831, + "tokens_seen": 999292928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521464393179539, + "loss": 2.7807, + "theoretical_loss": 3.6494124555285063, + "tokens_seen": 999358464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 559222, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.206585645675659, + "objective/train/theoretical_loss": 3.649389678284182, + "objective/train/tokens_used": 1019884000, + "theoretical_loss": 3.649389678284182, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352136409227683, + "loss": 2.775, + "theoretical_loss": 3.649389678284182, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035212637913741224, + "loss": 2.7452, + "theoretical_loss": 3.649366902951572, + "tokens_seen": 999489536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521163490471414, + "loss": 2.6143, + "theoretical_loss": 3.649344129530391, + "tokens_seen": 999555072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521063189568706, + "loss": 2.8037, + "theoretical_loss": 3.6493213580203534, + "tokens_seen": 999620608 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035209628886659984, + "loss": 2.6248, + "theoretical_loss": 3.6492985884211735, + "tokens_seen": 999686144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035208625877632897, + "loss": 3.0173, + "theoretical_loss": 3.6492758207325657, + "tokens_seen": 999751680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520762286860582, + "loss": 2.7024, + "theoretical_loss": 3.6492530549542437, + "tokens_seen": 999817216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520661985957874, + "loss": 2.943, + "theoretical_loss": 3.6492302910859227, + "tokens_seen": 999882752 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035205616850551657, + "loss": 3.0572, + "theoretical_loss": 3.6492075291273176, + "tokens_seen": 999948288 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035204613841524575, + "loss": 2.9415, + "theoretical_loss": 3.6491847690781425, + "tokens_seen": 1000013824 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352036108324975, + "loss": 2.6723, + "theoretical_loss": 3.649162010938112, + "tokens_seen": 1000079360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520260782347041, + "loss": 2.6927, + "theoretical_loss": 3.6491392547069412, + "tokens_seen": 1000144896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035201604814443334, + "loss": 2.7726, + "theoretical_loss": 3.649116500384345, + "tokens_seen": 1000210432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035200601805416247, + "loss": 2.7641, + "theoretical_loss": 3.6490937479700376, + "tokens_seen": 1000275968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519959879638917, + "loss": 2.7668, + "theoretical_loss": 3.649070997463735, + "tokens_seen": 1000341504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519859578736209, + "loss": 2.6113, + "theoretical_loss": 3.649048248865152, + "tokens_seen": 1000407040 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035197592778335007, + "loss": 2.6915, + "theoretical_loss": 3.649025502174003, + "tokens_seen": 1000472576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035196589769307925, + "loss": 2.7293, + "theoretical_loss": 3.649002757390003, + "tokens_seen": 1000538112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035195586760280843, + "loss": 2.4951, + "theoretical_loss": 3.648980014512868, + "tokens_seen": 1000603648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519458375125376, + "loss": 2.6436, + "theoretical_loss": 3.648957273542313, + "tokens_seen": 1000669184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035193580742226685, + "loss": 2.6603, + "theoretical_loss": 3.6489345344780535, + "tokens_seen": 1000734720 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351925777331996, + "loss": 2.6807, + "theoretical_loss": 3.648911797319804, + "tokens_seen": 1000800256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519157472417252, + "loss": 2.4684, + "theoretical_loss": 3.648889062067281, + "tokens_seen": 1000865792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035190571715145434, + "loss": 2.5266, + "theoretical_loss": 3.6488663287202, + "tokens_seen": 1000931328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035189568706118357, + "loss": 2.7564, + "theoretical_loss": 3.6488435972782756, + "tokens_seen": 1000996864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 559970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.601503372192383, + "objective/train/theoretical_loss": 3.6488208677412244, + "objective/train/tokens_used": 1021522400, + "theoretical_loss": 3.6488208677412244, + "tokens_seen": 1001062400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035188565697091275, + "loss": 2.6304, + "theoretical_loss": 3.6488208677412244, + "tokens_seen": 1001062400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035187562688064193, + "loss": 2.9246, + "theoretical_loss": 3.6487981401087612, + "tokens_seen": 1001127936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518655967903711, + "loss": 2.6429, + "theoretical_loss": 3.6487754143806024, + "tokens_seen": 1001193472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035185556670010035, + "loss": 2.6234, + "theoretical_loss": 3.6487526905564636, + "tokens_seen": 1001259008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518455366098295, + "loss": 2.7218, + "theoretical_loss": 3.64872996863606, + "tokens_seen": 1001324544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518355065195587, + "loss": 2.9734, + "theoretical_loss": 3.648707248619109, + "tokens_seen": 1001390080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035182547642928784, + "loss": 2.7579, + "theoretical_loss": 3.6486845305053253, + "tokens_seen": 1001455616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518154463390171, + "loss": 2.746, + "theoretical_loss": 3.6486618142944263, + "tokens_seen": 1001521152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035180541624874626, + "loss": 2.7835, + "theoretical_loss": 3.6486390999861262, + "tokens_seen": 1001586688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035179538615847544, + "loss": 2.7529, + "theoretical_loss": 3.6486163875801427, + "tokens_seen": 1001652224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517853560682046, + "loss": 2.6583, + "theoretical_loss": 3.6485936770761915, + "tokens_seen": 1001717760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517753259779338, + "loss": 2.7264, + "theoretical_loss": 3.6485709684739884, + "tokens_seen": 1001783296 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351765295887663, + "loss": 2.4917, + "theoretical_loss": 3.648548261773251, + "tokens_seen": 1001848832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517552657973922, + "loss": 2.6531, + "theoretical_loss": 3.648525556973695, + "tokens_seen": 1001914368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035174523570712134, + "loss": 2.809, + "theoretical_loss": 3.648502854075037, + "tokens_seen": 1001979904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517352056168506, + "loss": 2.6379, + "theoretical_loss": 3.648480153076993, + "tokens_seen": 1002045440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035172517552657976, + "loss": 2.6711, + "theoretical_loss": 3.64845745397928, + "tokens_seen": 1002110976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035171514543630894, + "loss": 2.7953, + "theoretical_loss": 3.6484347567816147, + "tokens_seen": 1002176512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517051153460381, + "loss": 2.6388, + "theoretical_loss": 3.648412061483714, + "tokens_seen": 1002242048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516950852557673, + "loss": 2.6233, + "theoretical_loss": 3.6483893680852946, + "tokens_seen": 1002307584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516850551654965, + "loss": 2.7684, + "theoretical_loss": 3.6483666765860727, + "tokens_seen": 1002373120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516750250752257, + "loss": 2.4296, + "theoretical_loss": 3.648343986985766, + "tokens_seen": 1002438656 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035166499498495485, + "loss": 2.7591, + "theoretical_loss": 3.6483212992840914, + "tokens_seen": 1002504192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516549648946841, + "loss": 2.5995, + "theoretical_loss": 3.6482986134807653, + "tokens_seen": 1002569728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516449348044132, + "loss": 2.9894, + "theoretical_loss": 3.6482759295755054, + "tokens_seen": 1002635264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 560865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5041370391845703, + "objective/train/theoretical_loss": 3.6482532475680287, + "objective/train/tokens_used": 1023160800, + "theoretical_loss": 3.6482532475680287, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035163490471414244, + "loss": 2.7673, + "theoretical_loss": 3.6482532475680287, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516248746238716, + "loss": 2.7331, + "theoretical_loss": 3.6482305674580524, + "tokens_seen": 1002766336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516148445336008, + "loss": 2.5743, + "theoretical_loss": 3.6482078892452936, + "tokens_seen": 1002831872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035160481444333, + "loss": 2.6882, + "theoretical_loss": 3.64818521292947, + "tokens_seen": 1002897408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035159478435305917, + "loss": 2.816, + "theoretical_loss": 3.648162538510298, + "tokens_seen": 1002962944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035158475426278835, + "loss": 2.6259, + "theoretical_loss": 3.6481398659874964, + "tokens_seen": 1003028480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515747241725176, + "loss": 2.8063, + "theoretical_loss": 3.648117195360782, + "tokens_seen": 1003094016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515646940822467, + "loss": 2.7044, + "theoretical_loss": 3.6480945266298725, + "tokens_seen": 1003159552 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035155466399197595, + "loss": 2.5904, + "theoretical_loss": 3.6480718597944852, + "tokens_seen": 1003225088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035154463390170513, + "loss": 2.7078, + "theoretical_loss": 3.648049194854338, + "tokens_seen": 1003290624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515346038114343, + "loss": 2.7708, + "theoretical_loss": 3.6480265318091485, + "tokens_seen": 1003356160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515245737211635, + "loss": 2.6009, + "theoretical_loss": 3.6480038706586355, + "tokens_seen": 1003421696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035151454363089267, + "loss": 2.866, + "theoretical_loss": 3.6479812114025156, + "tokens_seen": 1003487232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035150451354062185, + "loss": 2.6386, + "theoretical_loss": 3.6479585540405073, + "tokens_seen": 1003552768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514944834503511, + "loss": 2.6617, + "theoretical_loss": 3.6479358985723285, + "tokens_seen": 1003618304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514844533600802, + "loss": 2.7367, + "theoretical_loss": 3.647913244997697, + "tokens_seen": 1003683840 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035147442326980945, + "loss": 2.7202, + "theoretical_loss": 3.6478905933163315, + "tokens_seen": 1003749376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514643931795386, + "loss": 2.9854, + "theoretical_loss": 3.6478679435279497, + "tokens_seen": 1003814912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514543630892678, + "loss": 2.692, + "theoretical_loss": 3.6478452956322704, + "tokens_seen": 1003880448 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351444332998997, + "loss": 2.6885, + "theoretical_loss": 3.647822649629011, + "tokens_seen": 1003945984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514343029087262, + "loss": 2.9112, + "theoretical_loss": 3.6478000055178907, + "tokens_seen": 1004011520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035142427281845536, + "loss": 2.8827, + "theoretical_loss": 3.6477773632986272, + "tokens_seen": 1004077056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035141424272818454, + "loss": 2.5035, + "theoretical_loss": 3.6477547229709395, + "tokens_seen": 1004142592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514042126379137, + "loss": 2.6845, + "theoretical_loss": 3.647732084534546, + "tokens_seen": 1004208128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035139418254764295, + "loss": 2.8052, + "theoretical_loss": 3.647709447989165, + "tokens_seen": 1004273664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 561903, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.305278778076172, + "objective/train/theoretical_loss": 3.6476868133345155, + "objective/train/tokens_used": 1024799200, + "theoretical_loss": 3.6476868133345155, + "tokens_seen": 1004339200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513841524573721, + "loss": 2.7461, + "theoretical_loss": 3.6476868133345155, + "tokens_seen": 1004339200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513741223671013, + "loss": 2.505, + "theoretical_loss": 3.6476641805703163, + "tokens_seen": 1004404736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513640922768305, + "loss": 2.6162, + "theoretical_loss": 3.647641549696286, + "tokens_seen": 1004470272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513540621865597, + "loss": 2.715, + "theoretical_loss": 3.6476189207121434, + "tokens_seen": 1004535808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513440320962889, + "loss": 2.6644, + "theoretical_loss": 3.647596293617607, + "tokens_seen": 1004601344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035133400200601804, + "loss": 2.3282, + "theoretical_loss": 3.647573668412397, + "tokens_seen": 1004666880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513239719157473, + "loss": 2.8704, + "theoretical_loss": 3.6475510450962307, + "tokens_seen": 1004732416 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035131394182547646, + "loss": 2.9088, + "theoretical_loss": 3.6475284236688283, + "tokens_seen": 1004797952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035130391173520564, + "loss": 2.5969, + "theoretical_loss": 3.647505804129909, + "tokens_seen": 1004863488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512938816449348, + "loss": 2.7106, + "theoretical_loss": 3.647483186479192, + "tokens_seen": 1004929024 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351283851554664, + "loss": 2.6929, + "theoretical_loss": 3.6474605707163956, + "tokens_seen": 1004994560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512738214643932, + "loss": 2.7415, + "theoretical_loss": 3.64743795684124, + "tokens_seen": 1005060096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512637913741224, + "loss": 2.8185, + "theoretical_loss": 3.6474153448534445, + "tokens_seen": 1005125632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035125376128385154, + "loss": 2.6448, + "theoretical_loss": 3.6473927347527284, + "tokens_seen": 1005191168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512437311935808, + "loss": 2.8856, + "theoretical_loss": 3.647370126538811, + "tokens_seen": 1005256704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035123370110330996, + "loss": 2.484, + "theoretical_loss": 3.647347520211412, + "tokens_seen": 1005322240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035122367101303914, + "loss": 2.9744, + "theoretical_loss": 3.647324915770251, + "tokens_seen": 1005387776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512136409227683, + "loss": 2.7492, + "theoretical_loss": 3.6473023132150484, + "tokens_seen": 1005453312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512036108324975, + "loss": 2.5703, + "theoretical_loss": 3.6472797125455227, + "tokens_seen": 1005518848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511935807422267, + "loss": 2.8978, + "theoretical_loss": 3.6472571137613943, + "tokens_seen": 1005584384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511835506519559, + "loss": 2.9041, + "theoretical_loss": 3.647234516862383, + "tokens_seen": 1005649920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035117352056168505, + "loss": 2.6811, + "theoretical_loss": 3.647211921848209, + "tokens_seen": 1005715456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511634904714143, + "loss": 2.6016, + "theoretical_loss": 3.647189328718592, + "tokens_seen": 1005780992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511534603811434, + "loss": 2.8905, + "theoretical_loss": 3.6471667374732517, + "tokens_seen": 1005846528 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035114343029087264, + "loss": 2.6996, + "theoretical_loss": 3.6471441481119093, + "tokens_seen": 1005912064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 562546, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.896876335144043, + "objective/train/theoretical_loss": 3.6471215606342833, + "objective/train/tokens_used": 1026437600, + "theoretical_loss": 3.6471215606342833, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511334002006018, + "loss": 2.8747, + "theoretical_loss": 3.6471215606342833, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351123370110331, + "loss": 2.5018, + "theoretical_loss": 3.6470989750400955, + "tokens_seen": 1006043136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511133400200602, + "loss": 2.5671, + "theoretical_loss": 3.6470763913290654, + "tokens_seen": 1006108672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035110330992978937, + "loss": 2.6394, + "theoretical_loss": 3.647053809500913, + "tokens_seen": 1006174208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035109327983951855, + "loss": 2.7279, + "theoretical_loss": 3.6470312295553597, + "tokens_seen": 1006239744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510832497492478, + "loss": 2.6031, + "theoretical_loss": 3.6470086514921247, + "tokens_seen": 1006305280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510732196589769, + "loss": 2.6531, + "theoretical_loss": 3.6469860753109296, + "tokens_seen": 1006370816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035106318956870615, + "loss": 2.7165, + "theoretical_loss": 3.6469635010114945, + "tokens_seen": 1006436352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035105315947843533, + "loss": 2.9426, + "theoretical_loss": 3.64694092859354, + "tokens_seen": 1006501888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003510431293881645, + "loss": 2.9879, + "theoretical_loss": 3.646918358056787, + "tokens_seen": 1006567424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003510330992978937, + "loss": 2.6251, + "theoretical_loss": 3.6468957894009564, + "tokens_seen": 1006632960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035102306920762287, + "loss": 2.629, + "theoretical_loss": 3.6468732226257687, + "tokens_seen": 1006698496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035101303911735205, + "loss": 2.7709, + "theoretical_loss": 3.6468506577309445, + "tokens_seen": 1006764032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003510030090270813, + "loss": 2.7407, + "theoretical_loss": 3.6468280947162053, + "tokens_seen": 1006829568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509929789368104, + "loss": 2.6753, + "theoretical_loss": 3.6468055335812717, + "tokens_seen": 1006895104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035098294884653965, + "loss": 2.6529, + "theoretical_loss": 3.6467829743258653, + "tokens_seen": 1006960640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509729187562688, + "loss": 2.5762, + "theoretical_loss": 3.6467604169497063, + "tokens_seen": 1007026176 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350962888665998, + "loss": 2.9005, + "theoretical_loss": 3.6467378614525163, + "tokens_seen": 1007091712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509528585757272, + "loss": 2.7911, + "theoretical_loss": 3.6467153078340173, + "tokens_seen": 1007157248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509428284854564, + "loss": 2.5908, + "theoretical_loss": 3.6466927560939295, + "tokens_seen": 1007222784 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035093279839518556, + "loss": 2.6977, + "theoretical_loss": 3.6466702062319745, + "tokens_seen": 1007288320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035092276830491474, + "loss": 2.7994, + "theoretical_loss": 3.646647658247874, + "tokens_seen": 1007353856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509127382146439, + "loss": 2.5597, + "theoretical_loss": 3.6466251121413493, + "tokens_seen": 1007419392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035090270812437315, + "loss": 2.5898, + "theoretical_loss": 3.6466025679121223, + "tokens_seen": 1007484928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508926780341023, + "loss": 2.6098, + "theoretical_loss": 3.646580025559914, + "tokens_seen": 1007550464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 563033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2577614784240723, + "objective/train/theoretical_loss": 3.646557485084446, + "objective/train/tokens_used": 1028076000, + "theoretical_loss": 3.646557485084446, + "tokens_seen": 1007616000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508826479438315, + "loss": 2.7075, + "theoretical_loss": 3.646557485084446, + "tokens_seen": 1007616000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508726178535607, + "loss": 2.5445, + "theoretical_loss": 3.6465349464854406, + "tokens_seen": 1007681536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508625877632899, + "loss": 2.6022, + "theoretical_loss": 3.646512409762619, + "tokens_seen": 1007747072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035085255767301906, + "loss": 2.8948, + "theoretical_loss": 3.6464898749157033, + "tokens_seen": 1007812608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035084252758274824, + "loss": 2.8789, + "theoretical_loss": 3.646467341944416, + "tokens_seen": 1007878144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508324974924774, + "loss": 2.4769, + "theoretical_loss": 3.6464448108484775, + "tokens_seen": 1007943680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035082246740220666, + "loss": 2.6822, + "theoretical_loss": 3.6464222816276113, + "tokens_seen": 1008009216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508124373119358, + "loss": 2.6443, + "theoretical_loss": 3.6463997542815383, + "tokens_seen": 1008074752 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350802407221665, + "loss": 2.7836, + "theoretical_loss": 3.646377228809982, + "tokens_seen": 1008140288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035079237713139415, + "loss": 2.4362, + "theoretical_loss": 3.646354705212663, + "tokens_seen": 1008205824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507823470411234, + "loss": 2.6115, + "theoretical_loss": 3.6463321834893048, + "tokens_seen": 1008271360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035077231695085256, + "loss": 2.6302, + "theoretical_loss": 3.646309663639629, + "tokens_seen": 1008336896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035076228686058174, + "loss": 2.5671, + "theoretical_loss": 3.646287145663358, + "tokens_seen": 1008402432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507522567703109, + "loss": 2.6446, + "theoretical_loss": 3.6462646295602146, + "tokens_seen": 1008467968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035074222668004016, + "loss": 2.787, + "theoretical_loss": 3.6462421153299207, + "tokens_seen": 1008533504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507321965897693, + "loss": 2.3819, + "theoretical_loss": 3.6462196029721996, + "tokens_seen": 1008599040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507221664994985, + "loss": 2.6316, + "theoretical_loss": 3.6461970924867737, + "tokens_seen": 1008664576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035071213640922765, + "loss": 2.8287, + "theoretical_loss": 3.646174583873365, + "tokens_seen": 1008730112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507021063189569, + "loss": 2.8155, + "theoretical_loss": 3.6461520771316964, + "tokens_seen": 1008795648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035069207622868607, + "loss": 2.7533, + "theoretical_loss": 3.646129572261491, + "tokens_seen": 1008861184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035068204613841525, + "loss": 2.8465, + "theoretical_loss": 3.646107069262472, + "tokens_seen": 1008926720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035067201604814443, + "loss": 2.5893, + "theoretical_loss": 3.6460845681343614, + "tokens_seen": 1008992256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506619859578736, + "loss": 2.809, + "theoretical_loss": 3.646062068876883, + "tokens_seen": 1009057792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506519558676028, + "loss": 2.8392, + "theoretical_loss": 3.646039571489759, + "tokens_seen": 1009123328 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350641925777332, + "loss": 2.8639, + "theoretical_loss": 3.646017075972713, + "tokens_seen": 1009188864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 564080, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.776188373565674, + "objective/train/theoretical_loss": 3.645994582325468, + "objective/train/tokens_used": 1029714400, + "theoretical_loss": 3.645994582325468, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035063189568706115, + "loss": 2.9152, + "theoretical_loss": 3.645994582325468, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506218655967904, + "loss": 2.8531, + "theoretical_loss": 3.6459720905477466, + "tokens_seen": 1009319936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506118355065195, + "loss": 2.7551, + "theoretical_loss": 3.6459496006392733, + "tokens_seen": 1009385472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035060180541624875, + "loss": 2.5909, + "theoretical_loss": 3.6459271125997708, + "tokens_seen": 1009451008 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350591775325978, + "loss": 2.5893, + "theoretical_loss": 3.645904626428962, + "tokens_seen": 1009516544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505817452357071, + "loss": 2.863, + "theoretical_loss": 3.6458821421265704, + "tokens_seen": 1009582080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035057171514543635, + "loss": 2.9149, + "theoretical_loss": 3.64585965969232, + "tokens_seen": 1009647616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035056168505516553, + "loss": 2.4815, + "theoretical_loss": 3.6458371791259343, + "tokens_seen": 1009713152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505516549648947, + "loss": 2.8522, + "theoretical_loss": 3.6458147004271364, + "tokens_seen": 1009778688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505416248746239, + "loss": 2.4721, + "theoretical_loss": 3.6457922235956506, + "tokens_seen": 1009844224 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035053159478435307, + "loss": 2.6782, + "theoretical_loss": 3.6457697486312, + "tokens_seen": 1009909760 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035052156469408225, + "loss": 2.978, + "theoretical_loss": 3.645747275533509, + "tokens_seen": 1009975296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505115346038115, + "loss": 2.7994, + "theoretical_loss": 3.6457248043023007, + "tokens_seen": 1010040832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505015045135406, + "loss": 2.6268, + "theoretical_loss": 3.6457023349372992, + "tokens_seen": 1010106368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035049147442326985, + "loss": 2.7156, + "theoretical_loss": 3.645679867438229, + "tokens_seen": 1010171904 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350481444332999, + "loss": 2.7835, + "theoretical_loss": 3.6456574018048133, + "tokens_seen": 1010237440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504714142427282, + "loss": 2.8286, + "theoretical_loss": 3.6456349380367774, + "tokens_seen": 1010302976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504613841524574, + "loss": 2.8143, + "theoretical_loss": 3.6456124761338433, + "tokens_seen": 1010368512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504513540621866, + "loss": 2.613, + "theoretical_loss": 3.6455900160957375, + "tokens_seen": 1010434048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035044132397191576, + "loss": 2.484, + "theoretical_loss": 3.645567557922183, + "tokens_seen": 1010499584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035043129388164494, + "loss": 2.7924, + "theoretical_loss": 3.645545101612904, + "tokens_seen": 1010565120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504212637913741, + "loss": 2.7572, + "theoretical_loss": 3.645522647167626, + "tokens_seen": 1010630656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035041123370110335, + "loss": 2.6412, + "theoretical_loss": 3.645500194586072, + "tokens_seen": 1010696192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504012036108325, + "loss": 2.8669, + "theoretical_loss": 3.645477743867967, + "tokens_seen": 1010761728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503911735205617, + "loss": 2.8419, + "theoretical_loss": 3.645455295013036, + "tokens_seen": 1010827264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 564461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.960507869720459, + "objective/train/theoretical_loss": 3.6454328480210028, + "objective/train/tokens_used": 1031352800, + "theoretical_loss": 3.6454328480210028, + "tokens_seen": 1010892800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503811434302909, + "loss": 2.9234, + "theoretical_loss": 3.6454328480210028, + "tokens_seen": 1010892800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503711133400201, + "loss": 2.9007, + "theoretical_loss": 3.6454104028915926, + "tokens_seen": 1010958336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035036108324974926, + "loss": 2.8377, + "theoretical_loss": 3.64538795962453, + "tokens_seen": 1011023872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035035105315947844, + "loss": 2.7468, + "theoretical_loss": 3.64536551821954, + "tokens_seen": 1011089408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503410230692076, + "loss": 2.8693, + "theoretical_loss": 3.6453430786763468, + "tokens_seen": 1011154944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035033099297893686, + "loss": 2.4695, + "theoretical_loss": 3.645320640994676, + "tokens_seen": 1011220480 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350320962888666, + "loss": 2.6836, + "theoretical_loss": 3.645298205174252, + "tokens_seen": 1011286016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503109327983952, + "loss": 2.7281, + "theoretical_loss": 3.6452757712148, + "tokens_seen": 1011351552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035030090270812435, + "loss": 2.7976, + "theoretical_loss": 3.645253339116045, + "tokens_seen": 1011417088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502908726178536, + "loss": 2.6928, + "theoretical_loss": 3.6452309088777124, + "tokens_seen": 1011482624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035028084252758276, + "loss": 2.4886, + "theoretical_loss": 3.645208480499527, + "tokens_seen": 1011548160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035027081243731194, + "loss": 2.6801, + "theoretical_loss": 3.6451860539812144, + "tokens_seen": 1011613696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502607823470411, + "loss": 2.7794, + "theoretical_loss": 3.6451636293225, + "tokens_seen": 1011679232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035025075225677036, + "loss": 2.5784, + "theoretical_loss": 3.6451412065231086, + "tokens_seen": 1011744768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502407221664995, + "loss": 2.7135, + "theoretical_loss": 3.645118785582765, + "tokens_seen": 1011810304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502306920762287, + "loss": 2.6828, + "theoretical_loss": 3.6450963665011966, + "tokens_seen": 1011875840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035022066198595785, + "loss": 2.8697, + "theoretical_loss": 3.6450739492781277, + "tokens_seen": 1011941376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502106318956871, + "loss": 2.6464, + "theoretical_loss": 3.645051533913284, + "tokens_seen": 1012006912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035020060180541627, + "loss": 2.7643, + "theoretical_loss": 3.6450291204063907, + "tokens_seen": 1012072448 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035019057171514545, + "loss": 2.6983, + "theoretical_loss": 3.645006708757175, + "tokens_seen": 1012137984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035018054162487463, + "loss": 2.6446, + "theoretical_loss": 3.644984298965361, + "tokens_seen": 1012203520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501705115346038, + "loss": 2.5605, + "theoretical_loss": 3.6449618910306754, + "tokens_seen": 1012269056 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350160481444333, + "loss": 2.5893, + "theoretical_loss": 3.6449394849528436, + "tokens_seen": 1012334592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501504513540622, + "loss": 2.6389, + "theoretical_loss": 3.644917080731592, + "tokens_seen": 1012400128 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035014042126379135, + "loss": 2.7151, + "theoretical_loss": 3.6448946783666463, + "tokens_seen": 1012465664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 565625, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7371182441711426, + "objective/train/theoretical_loss": 3.6448722778577327, + "objective/train/tokens_used": 1032991200, + "theoretical_loss": 3.6448722778577327, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501303911735206, + "loss": 2.836, + "theoretical_loss": 3.6448722778577327, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501203610832497, + "loss": 2.6583, + "theoretical_loss": 3.6448498792045774, + "tokens_seen": 1012596736 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035011033099297895, + "loss": 2.5067, + "theoretical_loss": 3.6448274824069067, + "tokens_seen": 1012662272 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035010030090270813, + "loss": 2.6818, + "theoretical_loss": 3.644805087464446, + "tokens_seen": 1012727808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500902708124373, + "loss": 2.3187, + "theoretical_loss": 3.644782694376923, + "tokens_seen": 1012793344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500802407221665, + "loss": 2.6766, + "theoretical_loss": 3.6447603031440625, + "tokens_seen": 1012858880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035007021063189573, + "loss": 2.4468, + "theoretical_loss": 3.6447379137655913, + "tokens_seen": 1012924416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035006018054162486, + "loss": 2.6092, + "theoretical_loss": 3.644715526241237, + "tokens_seen": 1012989952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500501504513541, + "loss": 2.5698, + "theoretical_loss": 3.6446931405707246, + "tokens_seen": 1013055488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500401203610832, + "loss": 2.7784, + "theoretical_loss": 3.644670756753782, + "tokens_seen": 1013121024 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035003009027081245, + "loss": 2.5981, + "theoretical_loss": 3.6446483747901346, + "tokens_seen": 1013186560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035002006018054163, + "loss": 2.6603, + "theoretical_loss": 3.64462599467951, + "tokens_seen": 1013252096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500100300902708, + "loss": 2.9651, + "theoretical_loss": 3.644603616421634, + "tokens_seen": 1013317632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035, + "loss": 2.5475, + "theoretical_loss": 3.644581240016235, + "tokens_seen": 1013383168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499899699097292, + "loss": 2.9199, + "theoretical_loss": 3.644558865463038, + "tokens_seen": 1013448704 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034997993981945836, + "loss": 2.6318, + "theoretical_loss": 3.6445364927617714, + "tokens_seen": 1013514240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499699097291876, + "loss": 2.6628, + "theoretical_loss": 3.6445141219121617, + "tokens_seen": 1013579776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499598796389167, + "loss": 2.595, + "theoretical_loss": 3.6444917529139356, + "tokens_seen": 1013645312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034994984954864596, + "loss": 2.8204, + "theoretical_loss": 3.6444693857668207, + "tokens_seen": 1013710848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499398194583751, + "loss": 2.8346, + "theoretical_loss": 3.6444470204705435, + "tokens_seen": 1013776384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499297893681043, + "loss": 2.7842, + "theoretical_loss": 3.6444246570248318, + "tokens_seen": 1013841920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499197592778335, + "loss": 2.6016, + "theoretical_loss": 3.644402295429413, + "tokens_seen": 1013907456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499097291875627, + "loss": 2.818, + "theoretical_loss": 3.6443799356840136, + "tokens_seen": 1013972992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034989969909729186, + "loss": 2.5587, + "theoretical_loss": 3.6443575777883614, + "tokens_seen": 1014038528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498896690070211, + "loss": 2.6353, + "theoretical_loss": 3.6443352217421845, + "tokens_seen": 1014104064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 566086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7045507431030273, + "objective/train/theoretical_loss": 3.6443128675452092, + "objective/train/tokens_used": 1034629600, + "theoretical_loss": 3.6443128675452092, + "tokens_seen": 1014169600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498796389167502, + "loss": 2.6942, + "theoretical_loss": 3.6443128675452092, + "tokens_seen": 1014169600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034986960882647946, + "loss": 2.8548, + "theoretical_loss": 3.6442905151971647, + "tokens_seen": 1014235136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498595787362086, + "loss": 2.7197, + "theoretical_loss": 3.6442681646977766, + "tokens_seen": 1014300672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498495486459378, + "loss": 2.6459, + "theoretical_loss": 3.6442458160467743, + "tokens_seen": 1014366208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034983951855566706, + "loss": 2.5951, + "theoretical_loss": 3.644223469243885, + "tokens_seen": 1014431744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498294884653962, + "loss": 2.7552, + "theoretical_loss": 3.644201124288836, + "tokens_seen": 1014497280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498194583751254, + "loss": 2.7001, + "theoretical_loss": 3.6441787811813553, + "tokens_seen": 1014562816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034980942828485455, + "loss": 2.7129, + "theoretical_loss": 3.6441564399211708, + "tokens_seen": 1014628352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497993981945838, + "loss": 2.6051, + "theoretical_loss": 3.644134100508011, + "tokens_seen": 1014693888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034978936810431296, + "loss": 2.6101, + "theoretical_loss": 3.6441117629416038, + "tokens_seen": 1014759424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034977933801404214, + "loss": 2.6598, + "theoretical_loss": 3.644089427221677, + "tokens_seen": 1014824960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497693079237713, + "loss": 2.4322, + "theoretical_loss": 3.644067093347958, + "tokens_seen": 1014890496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034975927783350056, + "loss": 2.7851, + "theoretical_loss": 3.6440447613201767, + "tokens_seen": 1014956032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497492477432297, + "loss": 2.7539, + "theoretical_loss": 3.64402243113806, + "tokens_seen": 1015021568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497392176529589, + "loss": 2.5749, + "theoretical_loss": 3.6440001028013365, + "tokens_seen": 1015087104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034972918756268805, + "loss": 2.8158, + "theoretical_loss": 3.643977776309735, + "tokens_seen": 1015152640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497191574724173, + "loss": 2.753, + "theoretical_loss": 3.6439554516629835, + "tokens_seen": 1015218176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034970912738214647, + "loss": 2.7789, + "theoretical_loss": 3.6439331288608106, + "tokens_seen": 1015283712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034969909729187565, + "loss": 2.6908, + "theoretical_loss": 3.6439108079029445, + "tokens_seen": 1015349248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034968906720160483, + "loss": 2.742, + "theoretical_loss": 3.643888488789115, + "tokens_seen": 1015414784 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349679037111334, + "loss": 2.7681, + "theoretical_loss": 3.6438661715190492, + "tokens_seen": 1015480320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496690070210632, + "loss": 2.8472, + "theoretical_loss": 3.643843856092477, + "tokens_seen": 1015545856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496589769307924, + "loss": 2.5085, + "theoretical_loss": 3.6438215425091265, + "tokens_seen": 1015611392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034964894684052155, + "loss": 2.9066, + "theoretical_loss": 3.6437992307687264, + "tokens_seen": 1015676928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496389167502508, + "loss": 2.673, + "theoretical_loss": 3.6437769208710065, + "tokens_seen": 1015742464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 567051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.23244047164917, + "objective/train/theoretical_loss": 3.6437546128156946, + "objective/train/tokens_used": 1036268000, + "theoretical_loss": 3.6437546128156946, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496288866599799, + "loss": 2.4863, + "theoretical_loss": 3.6437546128156946, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034961885656970915, + "loss": 2.7115, + "theoretical_loss": 3.6437323066025207, + "tokens_seen": 1015873536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034960882647943833, + "loss": 2.6778, + "theoretical_loss": 3.643710002231213, + "tokens_seen": 1015939072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495987963891675, + "loss": 2.7031, + "theoretical_loss": 3.643687699701501, + "tokens_seen": 1016004608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495887662988967, + "loss": 2.8153, + "theoretical_loss": 3.6436653990131145, + "tokens_seen": 1016070144 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034957873620862593, + "loss": 2.6424, + "theoretical_loss": 3.6436431001657814, + "tokens_seen": 1016135680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034956870611835506, + "loss": 2.6089, + "theoretical_loss": 3.643620803159232, + "tokens_seen": 1016201216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495586760280843, + "loss": 2.6988, + "theoretical_loss": 3.643598507993195, + "tokens_seen": 1016266752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495486459378134, + "loss": 2.7106, + "theoretical_loss": 3.6435762146674007, + "tokens_seen": 1016332288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034953861584754265, + "loss": 2.4762, + "theoretical_loss": 3.6435539231815777, + "tokens_seen": 1016397824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034952858575727183, + "loss": 2.8292, + "theoretical_loss": 3.6435316335354555, + "tokens_seen": 1016463360 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349518555667001, + "loss": 2.6217, + "theoretical_loss": 3.6435093457287646, + "tokens_seen": 1016528896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495085255767302, + "loss": 2.7067, + "theoretical_loss": 3.6434870597612337, + "tokens_seen": 1016594432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494984954864594, + "loss": 2.7773, + "theoretical_loss": 3.6434647756325926, + "tokens_seen": 1016659968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034948846539618856, + "loss": 2.6261, + "theoretical_loss": 3.6434424933425715, + "tokens_seen": 1016725504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494784353059178, + "loss": 2.7, + "theoretical_loss": 3.6434202128909003, + "tokens_seen": 1016791040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494684052156469, + "loss": 2.7586, + "theoretical_loss": 3.6433979342773077, + "tokens_seen": 1016856576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034945837512537616, + "loss": 2.8934, + "theoretical_loss": 3.643375657501525, + "tokens_seen": 1016922112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494483450351053, + "loss": 2.6247, + "theoretical_loss": 3.6433533825632813, + "tokens_seen": 1016987648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494383149448345, + "loss": 2.8865, + "theoretical_loss": 3.6433311094623066, + "tokens_seen": 1017053184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494282848545637, + "loss": 2.6264, + "theoretical_loss": 3.643308838198332, + "tokens_seen": 1017118720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494182547642929, + "loss": 2.6972, + "theoretical_loss": 3.6432865687710865, + "tokens_seen": 1017184256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034940822467402206, + "loss": 2.7361, + "theoretical_loss": 3.643264301180301, + "tokens_seen": 1017249792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493981945837513, + "loss": 2.6566, + "theoretical_loss": 3.643242035425705, + "tokens_seen": 1017315328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493881644934804, + "loss": 2.6861, + "theoretical_loss": 3.64321977150703, + "tokens_seen": 1017380864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 568180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6473095417022705, + "objective/train/theoretical_loss": 3.643197509424005, + "objective/train/tokens_used": 1037906400, + "theoretical_loss": 3.643197509424005, + "tokens_seen": 1017446400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034937813440320966, + "loss": 2.7746, + "theoretical_loss": 3.643197509424005, + "tokens_seen": 1017446400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493681043129388, + "loss": 2.8067, + "theoretical_loss": 3.6431752491763616, + "tokens_seen": 1017511936 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349358074222668, + "loss": 2.6145, + "theoretical_loss": 3.64315299076383, + "tokens_seen": 1017577472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493480441323972, + "loss": 2.9663, + "theoretical_loss": 3.6431307341861396, + "tokens_seen": 1017643008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493380140421264, + "loss": 2.7876, + "theoretical_loss": 3.6431084794430224, + "tokens_seen": 1017708544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034932798395185557, + "loss": 2.7973, + "theoretical_loss": 3.6430862265342085, + "tokens_seen": 1017774080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034931795386158475, + "loss": 2.5958, + "theoretical_loss": 3.643063975459429, + "tokens_seen": 1017839616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034930792377131393, + "loss": 2.8849, + "theoretical_loss": 3.643041726218414, + "tokens_seen": 1017905152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034929789368104316, + "loss": 2.7891, + "theoretical_loss": 3.643019478810895, + "tokens_seen": 1017970688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492878635907723, + "loss": 2.7187, + "theoretical_loss": 3.6429972332366027, + "tokens_seen": 1018036224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492778335005015, + "loss": 2.7684, + "theoretical_loss": 3.6429749894952677, + "tokens_seen": 1018101760 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034926780341023065, + "loss": 2.3443, + "theoretical_loss": 3.6429527475866212, + "tokens_seen": 1018167296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492577733199599, + "loss": 2.8958, + "theoretical_loss": 3.6429305075103944, + "tokens_seen": 1018232832 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034924774322968907, + "loss": 2.7228, + "theoretical_loss": 3.642908269266318, + "tokens_seen": 1018298368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034923771313941825, + "loss": 2.9386, + "theoretical_loss": 3.6428860328541233, + "tokens_seen": 1018363904 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034922768304914743, + "loss": 2.6324, + "theoretical_loss": 3.6428637982735417, + "tokens_seen": 1018429440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034921765295887667, + "loss": 2.6251, + "theoretical_loss": 3.642841565524305, + "tokens_seen": 1018494976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492076228686058, + "loss": 2.6117, + "theoretical_loss": 3.6428193346061435, + "tokens_seen": 1018560512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034919759277833503, + "loss": 2.6193, + "theoretical_loss": 3.642797105518789, + "tokens_seen": 1018626048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034918756268806416, + "loss": 2.611, + "theoretical_loss": 3.6427748782619735, + "tokens_seen": 1018691584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491775325977934, + "loss": 2.5485, + "theoretical_loss": 3.6427526528354277, + "tokens_seen": 1018757120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034916750250752257, + "loss": 2.8654, + "theoretical_loss": 3.642730429238884, + "tokens_seen": 1018822656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034915747241725175, + "loss": 2.8412, + "theoretical_loss": 3.6427082074720727, + "tokens_seen": 1018888192 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034914744232698093, + "loss": 2.5741, + "theoretical_loss": 3.6426859875347266, + "tokens_seen": 1018953728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491374122367101, + "loss": 2.6678, + "theoretical_loss": 3.6426637694265773, + "tokens_seen": 1019019264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 568941, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.903310775756836, + "objective/train/theoretical_loss": 3.6426415531473566, + "objective/train/tokens_used": 1039544800, + "theoretical_loss": 3.6426415531473566, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491273821464393, + "loss": 2.701, + "theoretical_loss": 3.6426415531473566, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034911735205616853, + "loss": 2.8147, + "theoretical_loss": 3.6426193386967958, + "tokens_seen": 1019150336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034910732196589766, + "loss": 2.5661, + "theoretical_loss": 3.642597126074627, + "tokens_seen": 1019215872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490972918756269, + "loss": 2.8054, + "theoretical_loss": 3.6425749152805826, + "tokens_seen": 1019281408 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034908726178535613, + "loss": 2.7761, + "theoretical_loss": 3.6425527063143948, + "tokens_seen": 1019346944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034907723169508526, + "loss": 2.6967, + "theoretical_loss": 3.6425304991757943, + "tokens_seen": 1019412480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490672016048145, + "loss": 2.9256, + "theoretical_loss": 3.6425082938645144, + "tokens_seen": 1019478016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490571715145436, + "loss": 2.9273, + "theoretical_loss": 3.642486090380287, + "tokens_seen": 1019543552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034904714142427285, + "loss": 2.6534, + "theoretical_loss": 3.642463888722845, + "tokens_seen": 1019609088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034903711133400203, + "loss": 2.8481, + "theoretical_loss": 3.6424416888919193, + "tokens_seen": 1019674624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490270812437312, + "loss": 2.6814, + "theoretical_loss": 3.6424194908872436, + "tokens_seen": 1019740160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490170511534604, + "loss": 2.7889, + "theoretical_loss": 3.64239729470855, + "tokens_seen": 1019805696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490070210631896, + "loss": 2.8075, + "theoretical_loss": 3.6423751003555696, + "tokens_seen": 1019871232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034899699097291876, + "loss": 2.7066, + "theoretical_loss": 3.6423529078280366, + "tokens_seen": 1019936768 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348986960882648, + "loss": 2.6282, + "theoretical_loss": 3.642330717125683, + "tokens_seen": 1020002304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489769307923771, + "loss": 2.9868, + "theoretical_loss": 3.6423085282482415, + "tokens_seen": 1020067840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034896690070210636, + "loss": 2.8634, + "theoretical_loss": 3.6422863411954447, + "tokens_seen": 1020133376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489568706118355, + "loss": 2.7896, + "theoretical_loss": 3.642264155967026, + "tokens_seen": 1020198912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489468405215647, + "loss": 2.7245, + "theoretical_loss": 3.642241972562717, + "tokens_seen": 1020264448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489368104312939, + "loss": 2.7539, + "theoretical_loss": 3.642219790982251, + "tokens_seen": 1020329984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489267803410231, + "loss": 2.3399, + "theoretical_loss": 3.6421976112253613, + "tokens_seen": 1020395520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034891675025075226, + "loss": 2.5317, + "theoretical_loss": 3.64217543329178, + "tokens_seen": 1020461056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489067201604815, + "loss": 2.8228, + "theoretical_loss": 3.6421532571812416, + "tokens_seen": 1020526592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488966900702106, + "loss": 2.6505, + "theoretical_loss": 3.6421310828934783, + "tokens_seen": 1020592128 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034888665997993986, + "loss": 2.7931, + "theoretical_loss": 3.642108910428223, + "tokens_seen": 1020657664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 569860, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.728569507598877, + "objective/train/theoretical_loss": 3.6420867397852095, + "objective/train/tokens_used": 1041183200, + "theoretical_loss": 3.6420867397852095, + "tokens_seen": 1020723200 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348876629889669, + "loss": 2.6178, + "theoretical_loss": 3.6420867397852095, + "tokens_seen": 1020723200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488665997993982, + "loss": 2.7344, + "theoretical_loss": 3.6420645709641706, + "tokens_seen": 1020788736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488565697091274, + "loss": 2.82, + "theoretical_loss": 3.64204240396484, + "tokens_seen": 1020854272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488465396188566, + "loss": 2.7387, + "theoretical_loss": 3.6420202387869507, + "tokens_seen": 1020919808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034883650952858577, + "loss": 2.7402, + "theoretical_loss": 3.6419980754302363, + "tokens_seen": 1020985344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034882647943831495, + "loss": 2.8513, + "theoretical_loss": 3.64197591389443, + "tokens_seen": 1021050880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034881644934804413, + "loss": 2.9306, + "theoretical_loss": 3.6419537541792657, + "tokens_seen": 1021116416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034880641925777336, + "loss": 2.7919, + "theoretical_loss": 3.6419315962844774, + "tokens_seen": 1021181952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487963891675025, + "loss": 2.6795, + "theoretical_loss": 3.6419094402097976, + "tokens_seen": 1021247488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487863590772317, + "loss": 2.6768, + "theoretical_loss": 3.6418872859549607, + "tokens_seen": 1021313024 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034877632898696085, + "loss": 2.8067, + "theoretical_loss": 3.6418651335197003, + "tokens_seen": 1021378560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487662988966901, + "loss": 2.7436, + "theoretical_loss": 3.641842982903751, + "tokens_seen": 1021444096 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034875626880641927, + "loss": 2.8722, + "theoretical_loss": 3.641820834106845, + "tokens_seen": 1021509632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034874623871614845, + "loss": 2.8338, + "theoretical_loss": 3.641798687128718, + "tokens_seen": 1021575168 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034873620862587763, + "loss": 2.9625, + "theoretical_loss": 3.641776541969103, + "tokens_seen": 1021640704 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034872617853560687, + "loss": 2.5987, + "theoretical_loss": 3.641754398627734, + "tokens_seen": 1021706240 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348716148445336, + "loss": 2.882, + "theoretical_loss": 3.6417322571043456, + "tokens_seen": 1021771776 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034870611835506523, + "loss": 2.762, + "theoretical_loss": 3.641710117398672, + "tokens_seen": 1021837312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034869608826479436, + "loss": 2.5842, + "theoretical_loss": 3.641687979510446, + "tokens_seen": 1021902848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486860581745236, + "loss": 2.7492, + "theoretical_loss": 3.6416658434394042, + "tokens_seen": 1021968384 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034867602808425277, + "loss": 2.8195, + "theoretical_loss": 3.6416437091852787, + "tokens_seen": 1022033920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034866599799398195, + "loss": 2.9394, + "theoretical_loss": 3.6416215767478057, + "tokens_seen": 1022099456 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034865596790371113, + "loss": 2.5965, + "theoretical_loss": 3.641599446126718, + "tokens_seen": 1022164992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486459378134403, + "loss": 2.7538, + "theoretical_loss": 3.6415773173217514, + "tokens_seen": 1022230528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486359077231695, + "loss": 2.5377, + "theoretical_loss": 3.6415551903326397, + "tokens_seen": 1022296064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 570478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2970290184020996, + "objective/train/theoretical_loss": 3.641533065159118, + "objective/train/tokens_used": 1042821600, + "theoretical_loss": 3.641533065159118, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034862587763289873, + "loss": 2.6148, + "theoretical_loss": 3.641533065159118, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034861584754262786, + "loss": 2.8847, + "theoretical_loss": 3.6415109418009206, + "tokens_seen": 1022427136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486058174523571, + "loss": 2.7738, + "theoretical_loss": 3.6414888202577824, + "tokens_seen": 1022492672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485957873620862, + "loss": 2.9431, + "theoretical_loss": 3.6414667005294374, + "tokens_seen": 1022558208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034858575727181546, + "loss": 2.7978, + "theoretical_loss": 3.641444582615622, + "tokens_seen": 1022623744 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034857572718154464, + "loss": 2.7741, + "theoretical_loss": 3.6414224665160697, + "tokens_seen": 1022689280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485656970912738, + "loss": 2.6496, + "theoretical_loss": 3.641400352230516, + "tokens_seen": 1022754816 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348555667001003, + "loss": 2.6782, + "theoretical_loss": 3.6413782397586956, + "tokens_seen": 1022820352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034854563691073223, + "loss": 2.6638, + "theoretical_loss": 3.641356129100344, + "tokens_seen": 1022885888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034853560682046136, + "loss": 2.7403, + "theoretical_loss": 3.6413340202551954, + "tokens_seen": 1022951424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485255767301906, + "loss": 2.8024, + "theoretical_loss": 3.641311913222986, + "tokens_seen": 1023016960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485155466399197, + "loss": 2.6215, + "theoretical_loss": 3.6412898080034504, + "tokens_seen": 1023082496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034850551654964896, + "loss": 2.8317, + "theoretical_loss": 3.6412677045963244, + "tokens_seen": 1023148032 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034849548645937814, + "loss": 2.7537, + "theoretical_loss": 3.6412456030013427, + "tokens_seen": 1023213568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484854563691073, + "loss": 2.6645, + "theoretical_loss": 3.6412235032182414, + "tokens_seen": 1023279104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484754262788365, + "loss": 2.4196, + "theoretical_loss": 3.6412014052467554, + "tokens_seen": 1023344640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484653961885657, + "loss": 2.4108, + "theoretical_loss": 3.64117930908662, + "tokens_seen": 1023410176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034845536609829486, + "loss": 2.8733, + "theoretical_loss": 3.6411572147375715, + "tokens_seen": 1023475712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484453360080241, + "loss": 2.6105, + "theoretical_loss": 3.6411351221993447, + "tokens_seen": 1023541248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484353059177532, + "loss": 2.6572, + "theoretical_loss": 3.6411130314716758, + "tokens_seen": 1023606784 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034842527582748246, + "loss": 2.5466, + "theoretical_loss": 3.6410909425542997, + "tokens_seen": 1023672320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034841524573721164, + "loss": 2.493, + "theoretical_loss": 3.641068855446953, + "tokens_seen": 1023737856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484052156469408, + "loss": 2.6693, + "theoretical_loss": 3.641046770149372, + "tokens_seen": 1023803392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034839518555667, + "loss": 2.8194, + "theoretical_loss": 3.641024686661291, + "tokens_seen": 1023868928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483851554663992, + "loss": 2.6758, + "theoretical_loss": 3.6410026049824475, + "tokens_seen": 1023934464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 571810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.663266897201538, + "objective/train/theoretical_loss": 3.640980525112577, + "objective/train/tokens_used": 1044460000, + "theoretical_loss": 3.640980525112577, + "tokens_seen": 1024000000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034837512537612837, + "loss": 2.5753, + "theoretical_loss": 3.640980525112577, + "tokens_seen": 1024000000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483650952858576, + "loss": 2.7885, + "theoretical_loss": 3.640958447051415, + "tokens_seen": 1024065536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034835506519558673, + "loss": 2.5495, + "theoretical_loss": 3.6409363707986975, + "tokens_seen": 1024131072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034834503510531597, + "loss": 2.8806, + "theoretical_loss": 3.6409142963541616, + "tokens_seen": 1024196608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034833500501504515, + "loss": 2.8696, + "theoretical_loss": 3.6408922237175427, + "tokens_seen": 1024262144 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034832497492477433, + "loss": 2.6233, + "theoretical_loss": 3.640870152888578, + "tokens_seen": 1024327680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034831494483450356, + "loss": 2.7639, + "theoretical_loss": 3.6408480838670023, + "tokens_seen": 1024393216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483049147442327, + "loss": 2.8172, + "theoretical_loss": 3.6408260166525537, + "tokens_seen": 1024458752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482948846539619, + "loss": 2.7901, + "theoretical_loss": 3.6408039512449677, + "tokens_seen": 1024524288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034828485456369105, + "loss": 2.7927, + "theoretical_loss": 3.6407818876439806, + "tokens_seen": 1024589824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482748244734203, + "loss": 2.8879, + "theoretical_loss": 3.6407598258493294, + "tokens_seen": 1024655360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034826479438314947, + "loss": 2.7932, + "theoretical_loss": 3.6407377658607505, + "tokens_seen": 1024720896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034825476429287865, + "loss": 2.6392, + "theoretical_loss": 3.640715707677981, + "tokens_seen": 1024786432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034824473420260783, + "loss": 2.8363, + "theoretical_loss": 3.6406936513007566, + "tokens_seen": 1024851968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034823470411233707, + "loss": 2.6225, + "theoretical_loss": 3.640671596728815, + "tokens_seen": 1024917504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482246740220662, + "loss": 2.8723, + "theoretical_loss": 3.6406495439618927, + "tokens_seen": 1024983040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034821464393179543, + "loss": 2.6802, + "theoretical_loss": 3.6406274929997267, + "tokens_seen": 1025048576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034820461384152456, + "loss": 2.6781, + "theoretical_loss": 3.6406054438420536, + "tokens_seen": 1025114112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481945837512538, + "loss": 2.7748, + "theoretical_loss": 3.6405833964886103, + "tokens_seen": 1025179648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034818455366098297, + "loss": 2.6838, + "theoretical_loss": 3.6405613509391346, + "tokens_seen": 1025245184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034817452357071215, + "loss": 2.782, + "theoretical_loss": 3.640539307193363, + "tokens_seen": 1025310720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034816449348044133, + "loss": 2.7386, + "theoretical_loss": 3.640517265251033, + "tokens_seen": 1025376256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481544633901705, + "loss": 2.9111, + "theoretical_loss": 3.6404952251118816, + "tokens_seen": 1025441792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481444332998997, + "loss": 2.5696, + "theoretical_loss": 3.6404731867756457, + "tokens_seen": 1025507328 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034813440320962893, + "loss": 2.5349, + "theoretical_loss": 3.640451150242063, + "tokens_seen": 1025572864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 572959, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5865912437438965, + "objective/train/theoretical_loss": 3.6404291155108712, + "objective/train/tokens_used": 1046098400, + "theoretical_loss": 3.6404291155108712, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034812437311935806, + "loss": 2.5791, + "theoretical_loss": 3.6404291155108712, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481143430290873, + "loss": 2.7193, + "theoretical_loss": 3.640407082581807, + "tokens_seen": 1025703936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481043129388164, + "loss": 2.7651, + "theoretical_loss": 3.6403850514546083, + "tokens_seen": 1025769472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034809428284854566, + "loss": 2.6543, + "theoretical_loss": 3.6403630221290126, + "tokens_seen": 1025835008 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034808425275827484, + "loss": 2.7465, + "theoretical_loss": 3.640340994604758, + "tokens_seen": 1025900544 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348074222668004, + "loss": 2.5764, + "theoretical_loss": 3.640318968881581, + "tokens_seen": 1025966080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480641925777332, + "loss": 2.74, + "theoretical_loss": 3.6402969449592195, + "tokens_seen": 1026031616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034805416248746243, + "loss": 2.4407, + "theoretical_loss": 3.6402749228374125, + "tokens_seen": 1026097152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034804413239719156, + "loss": 2.5572, + "theoretical_loss": 3.6402529025158965, + "tokens_seen": 1026162688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480341023069208, + "loss": 2.9396, + "theoretical_loss": 3.64023088399441, + "tokens_seen": 1026228224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480240722166499, + "loss": 2.6433, + "theoretical_loss": 3.640208867272691, + "tokens_seen": 1026293760 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034801404212637916, + "loss": 2.8696, + "theoretical_loss": 3.6401868523504772, + "tokens_seen": 1026359296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034800401203610834, + "loss": 2.6728, + "theoretical_loss": 3.640164839227506, + "tokens_seen": 1026424832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479939819458375, + "loss": 2.6425, + "theoretical_loss": 3.640142827903517, + "tokens_seen": 1026490368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479839518555667, + "loss": 2.7012, + "theoretical_loss": 3.6401208183782474, + "tokens_seen": 1026555904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479739217652959, + "loss": 2.7336, + "theoretical_loss": 3.640098810651435, + "tokens_seen": 1026621440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034796389167502506, + "loss": 2.8522, + "theoretical_loss": 3.6400768047228187, + "tokens_seen": 1026686976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479538615847543, + "loss": 2.7545, + "theoretical_loss": 3.6400548005921367, + "tokens_seen": 1026752512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034794383149448343, + "loss": 2.6936, + "theoretical_loss": 3.6400327982591274, + "tokens_seen": 1026818048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034793380140421266, + "loss": 2.7252, + "theoretical_loss": 3.640010797723529, + "tokens_seen": 1026883584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034792377131394184, + "loss": 2.9514, + "theoretical_loss": 3.6399887989850805, + "tokens_seen": 1026949120 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347913741223671, + "loss": 2.6321, + "theoretical_loss": 3.63996680204352, + "tokens_seen": 1027014656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479037111334002, + "loss": 2.6391, + "theoretical_loss": 3.639944806898585, + "tokens_seen": 1027080192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478936810431294, + "loss": 2.6399, + "theoretical_loss": 3.639922813550016, + "tokens_seen": 1027145728 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034788365095285857, + "loss": 2.8079, + "theoretical_loss": 3.6399008219975513, + "tokens_seen": 1027211264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 573554, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3572163581848145, + "objective/train/theoretical_loss": 3.639878832240929, + "objective/train/tokens_used": 1047736800, + "theoretical_loss": 3.639878832240929, + "tokens_seen": 1027276800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478736208625878, + "loss": 2.6548, + "theoretical_loss": 3.639878832240929, + "tokens_seen": 1027276800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034786359077231693, + "loss": 2.6283, + "theoretical_loss": 3.6398568442798878, + "tokens_seen": 1027342336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034785356068204617, + "loss": 2.7042, + "theoretical_loss": 3.6398348581141677, + "tokens_seen": 1027407872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478435305917753, + "loss": 2.6247, + "theoretical_loss": 3.639812873743506, + "tokens_seen": 1027473408 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034783350050150453, + "loss": 2.6911, + "theoretical_loss": 3.6397908911676424, + "tokens_seen": 1027538944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478234704112337, + "loss": 2.8631, + "theoretical_loss": 3.6397689103863167, + "tokens_seen": 1027604480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478134403209629, + "loss": 2.7386, + "theoretical_loss": 3.6397469313992667, + "tokens_seen": 1027670016 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034780341023069207, + "loss": 2.7186, + "theoretical_loss": 3.6397249542062324, + "tokens_seen": 1027735552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034779338014042125, + "loss": 2.8693, + "theoretical_loss": 3.6397029788069526, + "tokens_seen": 1027801088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034778335005015043, + "loss": 2.5519, + "theoretical_loss": 3.639681005201167, + "tokens_seen": 1027866624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034777331995987967, + "loss": 2.6078, + "theoretical_loss": 3.6396590333886136, + "tokens_seen": 1027932160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477632898696088, + "loss": 2.5822, + "theoretical_loss": 3.639637063369033, + "tokens_seen": 1027997696 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034775325977933803, + "loss": 2.8317, + "theoretical_loss": 3.6396150951421644, + "tokens_seen": 1028063232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477432296890672, + "loss": 2.6957, + "theoretical_loss": 3.639593128707747, + "tokens_seen": 1028128768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477331995987964, + "loss": 2.6182, + "theoretical_loss": 3.6395711640655204, + "tokens_seen": 1028194304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477231695085256, + "loss": 2.5697, + "theoretical_loss": 3.639549201215224, + "tokens_seen": 1028259840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034771313941825476, + "loss": 2.4462, + "theoretical_loss": 3.639527240156598, + "tokens_seen": 1028325376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034770310932798394, + "loss": 2.7429, + "theoretical_loss": 3.6395052808893813, + "tokens_seen": 1028390912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034769307923771317, + "loss": 2.8586, + "theoretical_loss": 3.639483323413314, + "tokens_seen": 1028456448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476830491474423, + "loss": 2.8388, + "theoretical_loss": 3.6394613677281358, + "tokens_seen": 1028521984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034767301905717153, + "loss": 2.7639, + "theoretical_loss": 3.6394394138335864, + "tokens_seen": 1028587520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034766298896690066, + "loss": 2.7978, + "theoretical_loss": 3.639417461729406, + "tokens_seen": 1028653056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476529588766299, + "loss": 2.6436, + "theoretical_loss": 3.6393955114153345, + "tokens_seen": 1028718592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476429287863591, + "loss": 2.7939, + "theoretical_loss": 3.6393735628911115, + "tokens_seen": 1028784128 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034763289869608826, + "loss": 2.563, + "theoretical_loss": 3.639351616156478, + "tokens_seen": 1028849664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 574441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6161293983459473, + "objective/train/theoretical_loss": 3.639329671211173, + "objective/train/tokens_used": 1049375200, + "theoretical_loss": 3.639329671211173, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034762286860581744, + "loss": 2.5903, + "theoretical_loss": 3.639329671211173, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476128385155466, + "loss": 2.5422, + "theoretical_loss": 3.639307728054937, + "tokens_seen": 1028980736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476028084252758, + "loss": 2.5475, + "theoretical_loss": 3.639285786687511, + "tokens_seen": 1029046272 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034759277833500504, + "loss": 3.0258, + "theoretical_loss": 3.6392638471086336, + "tokens_seen": 1029111808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475827482447342, + "loss": 2.5871, + "theoretical_loss": 3.639241909318047, + "tokens_seen": 1029177344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475727181544634, + "loss": 2.6932, + "theoretical_loss": 3.6392199733154897, + "tokens_seen": 1029242880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034756268806419263, + "loss": 2.6994, + "theoretical_loss": 3.6391980391007035, + "tokens_seen": 1029308416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034755265797392176, + "loss": 2.6649, + "theoretical_loss": 3.639176106673429, + "tokens_seen": 1029373952 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347542627883651, + "loss": 2.4648, + "theoretical_loss": 3.639154176033406, + "tokens_seen": 1029439488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475325977933801, + "loss": 2.7083, + "theoretical_loss": 3.639132247180375, + "tokens_seen": 1029505024 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034752256770310936, + "loss": 2.6017, + "theoretical_loss": 3.639110320114077, + "tokens_seen": 1029570560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034751253761283854, + "loss": 2.6896, + "theoretical_loss": 3.6390883948342534, + "tokens_seen": 1029636096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475025075225677, + "loss": 2.811, + "theoretical_loss": 3.6390664713406435, + "tokens_seen": 1029701632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474924774322969, + "loss": 2.5898, + "theoretical_loss": 3.639044549632989, + "tokens_seen": 1029767168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474824473420261, + "loss": 2.7057, + "theoretical_loss": 3.6390226297110306, + "tokens_seen": 1029832704 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034747241725175527, + "loss": 2.2744, + "theoretical_loss": 3.6390007115745098, + "tokens_seen": 1029898240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474623871614845, + "loss": 2.6949, + "theoretical_loss": 3.6389787952231663, + "tokens_seen": 1029963776 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034745235707121363, + "loss": 2.5461, + "theoretical_loss": 3.6389568806567416, + "tokens_seen": 1030029312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034744232698094286, + "loss": 2.6111, + "theoretical_loss": 3.638934967874978, + "tokens_seen": 1030094848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034743229689067204, + "loss": 2.8444, + "theoretical_loss": 3.638913056877615, + "tokens_seen": 1030160384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474222668004012, + "loss": 2.7156, + "theoretical_loss": 3.6388911476643946, + "tokens_seen": 1030225920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474122367101304, + "loss": 2.7605, + "theoretical_loss": 3.6388692402350578, + "tokens_seen": 1030291456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474022066198596, + "loss": 2.578, + "theoretical_loss": 3.6388473345893457, + "tokens_seen": 1030356992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034739217652958877, + "loss": 2.6943, + "theoretical_loss": 3.638825430727, + "tokens_seen": 1030422528 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347382146439318, + "loss": 3.0443, + "theoretical_loss": 3.6388035286477622, + "tokens_seen": 1030488064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 574891, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2433831691741943, + "objective/train/theoretical_loss": 3.6387816283513734, + "objective/train/tokens_used": 1051013600, + "theoretical_loss": 3.6387816283513734, + "tokens_seen": 1030553600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034737211634904713, + "loss": 2.8729, + "theoretical_loss": 3.6387816283513734, + "tokens_seen": 1030553600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034736208625877637, + "loss": 2.4595, + "theoretical_loss": 3.6387597298375756, + "tokens_seen": 1030619136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473520561685055, + "loss": 2.7197, + "theoretical_loss": 3.63873783310611, + "tokens_seen": 1030684672 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034734202607823473, + "loss": 2.8291, + "theoretical_loss": 3.638715938156718, + "tokens_seen": 1030750208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473319959879639, + "loss": 2.9111, + "theoretical_loss": 3.638694044989142, + "tokens_seen": 1030815744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473219658976931, + "loss": 2.5698, + "theoretical_loss": 3.638672153603123, + "tokens_seen": 1030881280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034731193580742227, + "loss": 2.6734, + "theoretical_loss": 3.638650263998403, + "tokens_seen": 1030946816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034730190571715145, + "loss": 2.8147, + "theoretical_loss": 3.638628376174724, + "tokens_seen": 1031012352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034729187562688063, + "loss": 2.7605, + "theoretical_loss": 3.638606490131828, + "tokens_seen": 1031077888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034728184553660987, + "loss": 2.5385, + "theoretical_loss": 3.6385846058694558, + "tokens_seen": 1031143424 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347271815446339, + "loss": 2.8986, + "theoretical_loss": 3.638562723387351, + "tokens_seen": 1031208960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034726178535606823, + "loss": 2.7973, + "theoretical_loss": 3.638540842685255, + "tokens_seen": 1031274496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472517552657974, + "loss": 2.6124, + "theoretical_loss": 3.63851896376291, + "tokens_seen": 1031340032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472417251755266, + "loss": 2.5257, + "theoretical_loss": 3.6384970866200574, + "tokens_seen": 1031405568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472316950852558, + "loss": 2.7745, + "theoretical_loss": 3.6384752112564405, + "tokens_seen": 1031471104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034722166499498496, + "loss": 2.8269, + "theoretical_loss": 3.638453337671801, + "tokens_seen": 1031536640 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034721163490471414, + "loss": 2.6715, + "theoretical_loss": 3.638431465865881, + "tokens_seen": 1031602176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034720160481444337, + "loss": 2.8328, + "theoretical_loss": 3.6384095958384237, + "tokens_seen": 1031667712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471915747241725, + "loss": 2.7759, + "theoretical_loss": 3.638387727589171, + "tokens_seen": 1031733248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034718154463390173, + "loss": 3.0587, + "theoretical_loss": 3.6383658611178653, + "tokens_seen": 1031798784 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034717151454363086, + "loss": 2.4092, + "theoretical_loss": 3.6383439964242488, + "tokens_seen": 1031864320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471614844533601, + "loss": 2.5852, + "theoretical_loss": 3.638322133508065, + "tokens_seen": 1031929856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471514543630893, + "loss": 2.6847, + "theoretical_loss": 3.6383002723690563, + "tokens_seen": 1031995392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034714142427281846, + "loss": 2.7011, + "theoretical_loss": 3.6382784130069643, + "tokens_seen": 1032060928 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034713139418254764, + "loss": 2.8238, + "theoretical_loss": 3.6382565554215334, + "tokens_seen": 1032126464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 576151, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7170889377593994, + "objective/train/theoretical_loss": 3.6382346996125055, + "objective/train/tokens_used": 1052652000, + "theoretical_loss": 3.6382346996125055, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471213640922768, + "loss": 2.6599, + "theoretical_loss": 3.6382346996125055, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347111334002006, + "loss": 2.6971, + "theoretical_loss": 3.638212845579623, + "tokens_seen": 1032257536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034710130391173524, + "loss": 2.6737, + "theoretical_loss": 3.63819099332263, + "tokens_seen": 1032323072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034709127382146436, + "loss": 2.8611, + "theoretical_loss": 3.6381691428412686, + "tokens_seen": 1032388608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470812437311936, + "loss": 2.5908, + "theoretical_loss": 3.6381472941352815, + "tokens_seen": 1032454144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470712136409228, + "loss": 2.7545, + "theoretical_loss": 3.638125447204413, + "tokens_seen": 1032519680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034706118355065196, + "loss": 2.8943, + "theoretical_loss": 3.6381036020484054, + "tokens_seen": 1032585216 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034705115346038114, + "loss": 2.7134, + "theoretical_loss": 3.638081758667002, + "tokens_seen": 1032650752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470411233701103, + "loss": 2.6238, + "theoretical_loss": 3.6380599170599464, + "tokens_seen": 1032716288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470310932798395, + "loss": 2.7772, + "theoretical_loss": 3.6380380772269807, + "tokens_seen": 1032781824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034702106318956874, + "loss": 2.7171, + "theoretical_loss": 3.6380162391678494, + "tokens_seen": 1032847360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034701103309929787, + "loss": 2.7668, + "theoretical_loss": 3.637994402882296, + "tokens_seen": 1032912896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470010030090271, + "loss": 2.5211, + "theoretical_loss": 3.637972568370063, + "tokens_seen": 1032978432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034699097291875623, + "loss": 2.7032, + "theoretical_loss": 3.6379507356308944, + "tokens_seen": 1033043968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034698094282848547, + "loss": 2.818, + "theoretical_loss": 3.637928904664534, + "tokens_seen": 1033109504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034697091273821465, + "loss": 2.4894, + "theoretical_loss": 3.637907075470725, + "tokens_seen": 1033175040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034696088264794383, + "loss": 2.7589, + "theoretical_loss": 3.637885248049211, + "tokens_seen": 1033240576 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346950852557673, + "loss": 2.5823, + "theoretical_loss": 3.6378634223997364, + "tokens_seen": 1033306112 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034694082246740224, + "loss": 2.6596, + "theoretical_loss": 3.6378415985220443, + "tokens_seen": 1033371648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034693079237713137, + "loss": 2.7211, + "theoretical_loss": 3.6378197764158786, + "tokens_seen": 1033437184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003469207622868606, + "loss": 2.4441, + "theoretical_loss": 3.637797956080983, + "tokens_seen": 1033502720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034691073219658973, + "loss": 2.701, + "theoretical_loss": 3.6377761375171023, + "tokens_seen": 1033568256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034690070210631897, + "loss": 2.5034, + "theoretical_loss": 3.637754320723979, + "tokens_seen": 1033633792 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034689067201604815, + "loss": 2.615, + "theoretical_loss": 3.6377325057013588, + "tokens_seen": 1033699328 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034688064192577733, + "loss": 2.6485, + "theoretical_loss": 3.6377106924489846, + "tokens_seen": 1033764864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 577510, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.366300106048584, + "objective/train/theoretical_loss": 3.637688880966601, + "objective/train/tokens_used": 1054290400, + "theoretical_loss": 3.637688880966601, + "tokens_seen": 1033830400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468706118355065, + "loss": 2.4252, + "theoretical_loss": 3.637688880966601, + "tokens_seen": 1033830400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468605817452357, + "loss": 2.768, + "theoretical_loss": 3.6376670712539516, + "tokens_seen": 1033895936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468505516549649, + "loss": 2.7405, + "theoretical_loss": 3.6376452633107816, + "tokens_seen": 1033961472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468405215646941, + "loss": 2.7376, + "theoretical_loss": 3.637623457136835, + "tokens_seen": 1034027008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468304914744233, + "loss": 2.6507, + "theoretical_loss": 3.6376016527318558, + "tokens_seen": 1034092544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034682046138415247, + "loss": 2.808, + "theoretical_loss": 3.6375798500955887, + "tokens_seen": 1034158080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034681043129388165, + "loss": 2.5972, + "theoretical_loss": 3.6375580492277777, + "tokens_seen": 1034223616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034680040120361083, + "loss": 2.8287, + "theoretical_loss": 3.637536250128168, + "tokens_seen": 1034289152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034679037111334007, + "loss": 2.6991, + "theoretical_loss": 3.6375144527965038, + "tokens_seen": 1034354688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467803410230692, + "loss": 2.8229, + "theoretical_loss": 3.6374926572325297, + "tokens_seen": 1034420224 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034677031093279843, + "loss": 2.7673, + "theoretical_loss": 3.63747086343599, + "tokens_seen": 1034485760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467602808425276, + "loss": 2.7937, + "theoretical_loss": 3.63744907140663, + "tokens_seen": 1034551296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467502507522568, + "loss": 2.7354, + "theoretical_loss": 3.637427281144195, + "tokens_seen": 1034616832 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346740220661986, + "loss": 2.7832, + "theoretical_loss": 3.637405492648428, + "tokens_seen": 1034682368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034673019057171516, + "loss": 2.7563, + "theoretical_loss": 3.637383705919076, + "tokens_seen": 1034747904 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034672016048144434, + "loss": 2.7772, + "theoretical_loss": 3.637361920955882, + "tokens_seen": 1034813440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034671013039117357, + "loss": 2.7117, + "theoretical_loss": 3.637340137758592, + "tokens_seen": 1034878976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467001003009027, + "loss": 2.6319, + "theoretical_loss": 3.6373183563269516, + "tokens_seen": 1034944512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034669007021063193, + "loss": 2.608, + "theoretical_loss": 3.637296576660705, + "tokens_seen": 1035010048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034668004012036106, + "loss": 2.5727, + "theoretical_loss": 3.6372747987595977, + "tokens_seen": 1035075584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466700100300903, + "loss": 2.7038, + "theoretical_loss": 3.6372530226233746, + "tokens_seen": 1035141120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466599799398195, + "loss": 2.7065, + "theoretical_loss": 3.637231248251781, + "tokens_seen": 1035206656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034664994984954866, + "loss": 2.9293, + "theoretical_loss": 3.6372094756445623, + "tokens_seen": 1035272192 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034663991975927784, + "loss": 2.7916, + "theoretical_loss": 3.6371877048014642, + "tokens_seen": 1035337728 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346629889669007, + "loss": 2.5348, + "theoretical_loss": 3.6371659357222317, + "tokens_seen": 1035403264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 578118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6748743057250977, + "objective/train/theoretical_loss": 3.6371441684066097, + "objective/train/tokens_used": 1055928800, + "theoretical_loss": 3.6371441684066097, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466198595787362, + "loss": 2.7133, + "theoretical_loss": 3.6371441684066097, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034660982948846544, + "loss": 2.5092, + "theoretical_loss": 3.637122402854345, + "tokens_seen": 1035534336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034659979939819456, + "loss": 2.6481, + "theoretical_loss": 3.6371006390651828, + "tokens_seen": 1035599872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465897693079238, + "loss": 2.7212, + "theoretical_loss": 3.637078877038868, + "tokens_seen": 1035665408 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346579739217653, + "loss": 2.4035, + "theoretical_loss": 3.6370571167751473, + "tokens_seen": 1035730944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034656970912738216, + "loss": 2.4784, + "theoretical_loss": 3.6370353582737653, + "tokens_seen": 1035796480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034655967903711134, + "loss": 2.5488, + "theoretical_loss": 3.6370136015344685, + "tokens_seen": 1035862016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465496489468405, + "loss": 2.713, + "theoretical_loss": 3.6369918465570024, + "tokens_seen": 1035927552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465396188565697, + "loss": 2.7944, + "theoretical_loss": 3.6369700933411133, + "tokens_seen": 1035993088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034652958876629894, + "loss": 2.6355, + "theoretical_loss": 3.636948341886547, + "tokens_seen": 1036058624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034651955867602807, + "loss": 2.6931, + "theoretical_loss": 3.636926592193049, + "tokens_seen": 1036124160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465095285857573, + "loss": 2.761, + "theoretical_loss": 3.636904844260366, + "tokens_seen": 1036189696 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034649949849548643, + "loss": 2.7571, + "theoretical_loss": 3.636883098088244, + "tokens_seen": 1036255232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034648946840521567, + "loss": 2.7743, + "theoretical_loss": 3.6368613536764283, + "tokens_seen": 1036320768 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034647943831494485, + "loss": 2.2489, + "theoretical_loss": 3.6368396110246666, + "tokens_seen": 1036386304 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034646940822467403, + "loss": 2.7951, + "theoretical_loss": 3.6368178701327034, + "tokens_seen": 1036451840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464593781344032, + "loss": 2.5367, + "theoretical_loss": 3.6367961310002865, + "tokens_seen": 1036517376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034644934804413244, + "loss": 2.635, + "theoretical_loss": 3.636774393627162, + "tokens_seen": 1036582912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034643931795386157, + "loss": 2.563, + "theoretical_loss": 3.6367526580130756, + "tokens_seen": 1036648448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464292878635908, + "loss": 2.4748, + "theoretical_loss": 3.6367309241577743, + "tokens_seen": 1036713984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034641925777331993, + "loss": 2.7516, + "theoretical_loss": 3.636709192061004, + "tokens_seen": 1036779520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034640922768304917, + "loss": 2.6484, + "theoretical_loss": 3.636687461722512, + "tokens_seen": 1036845056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034639919759277835, + "loss": 2.8035, + "theoretical_loss": 3.6366657331420447, + "tokens_seen": 1036910592 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034638916750250753, + "loss": 2.6139, + "theoretical_loss": 3.636644006319349, + "tokens_seen": 1036976128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463791374122367, + "loss": 2.7835, + "theoretical_loss": 3.636622281254171, + "tokens_seen": 1037041664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 579340, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.744521379470825, + "objective/train/theoretical_loss": 3.6366005579462577, + "objective/train/tokens_used": 1057567200, + "theoretical_loss": 3.6366005579462577, + "tokens_seen": 1037107200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463691073219659, + "loss": 2.6837, + "theoretical_loss": 3.6366005579462577, + "tokens_seen": 1037107200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463590772316951, + "loss": 2.7886, + "theoretical_loss": 3.6365788363953566, + "tokens_seen": 1037172736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463490471414243, + "loss": 2.566, + "theoretical_loss": 3.636557116601214, + "tokens_seen": 1037238272 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034633901705115344, + "loss": 2.6344, + "theoretical_loss": 3.636535398563576, + "tokens_seen": 1037303808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034632898696088267, + "loss": 2.6984, + "theoretical_loss": 3.6365136822821915, + "tokens_seen": 1037369344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463189568706118, + "loss": 2.5895, + "theoretical_loss": 3.636491967756806, + "tokens_seen": 1037434880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034630892678034103, + "loss": 2.8962, + "theoretical_loss": 3.6364702549871675, + "tokens_seen": 1037500416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462988966900702, + "loss": 2.7219, + "theoretical_loss": 3.6364485439730223, + "tokens_seen": 1037565952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462888665997994, + "loss": 2.6679, + "theoretical_loss": 3.6364268347141184, + "tokens_seen": 1037631488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462788365095286, + "loss": 2.7464, + "theoretical_loss": 3.6364051272102027, + "tokens_seen": 1037697024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462688064192578, + "loss": 2.825, + "theoretical_loss": 3.636383421461022, + "tokens_seen": 1037762560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034625877632898694, + "loss": 2.603, + "theoretical_loss": 3.6363617174663245, + "tokens_seen": 1037828096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462487462387162, + "loss": 2.7439, + "theoretical_loss": 3.6363400152258576, + "tokens_seen": 1037893632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462387161484453, + "loss": 2.6634, + "theoretical_loss": 3.636318314739368, + "tokens_seen": 1037959168 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034622868605817454, + "loss": 2.6705, + "theoretical_loss": 3.6362966160066037, + "tokens_seen": 1038024704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462186559679037, + "loss": 2.5983, + "theoretical_loss": 3.6362749190273123, + "tokens_seen": 1038090240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462086258776329, + "loss": 2.7591, + "theoretical_loss": 3.6362532238012415, + "tokens_seen": 1038155776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461985957873621, + "loss": 2.3577, + "theoretical_loss": 3.6362315303281383, + "tokens_seen": 1038221312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034618856569709126, + "loss": 2.5005, + "theoretical_loss": 3.6362098386077513, + "tokens_seen": 1038286848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034617853560682044, + "loss": 2.6127, + "theoretical_loss": 3.636188148639828, + "tokens_seen": 1038352384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461685055165497, + "loss": 2.8448, + "theoretical_loss": 3.636166460424116, + "tokens_seen": 1038417920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461584754262788, + "loss": 2.3289, + "theoretical_loss": 3.636144773960363, + "tokens_seen": 1038483456 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034614844533600804, + "loss": 2.5375, + "theoretical_loss": 3.6361230892483176, + "tokens_seen": 1038548992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034613841524573717, + "loss": 2.7036, + "theoretical_loss": 3.636101406287727, + "tokens_seen": 1038614528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461283851554664, + "loss": 2.4236, + "theoretical_loss": 3.63607972507834, + "tokens_seen": 1038680064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 579796, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.493173837661743, + "objective/train/theoretical_loss": 3.6360580456199036, + "objective/train/tokens_used": 1059205600, + "theoretical_loss": 3.6360580456199036, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461183550651956, + "loss": 2.5022, + "theoretical_loss": 3.6360580456199036, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034610832497492476, + "loss": 2.841, + "theoretical_loss": 3.636036367912167, + "tokens_seen": 1038811136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034609829488465395, + "loss": 2.7636, + "theoretical_loss": 3.6360146919548786, + "tokens_seen": 1038876672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460882647943832, + "loss": 2.8077, + "theoretical_loss": 3.6359930177477855, + "tokens_seen": 1038942208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034607823470411236, + "loss": 2.6653, + "theoretical_loss": 3.635971345290637, + "tokens_seen": 1039007744 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034606820461384154, + "loss": 2.5918, + "theoretical_loss": 3.635949674583181, + "tokens_seen": 1039073280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460581745235707, + "loss": 2.6781, + "theoretical_loss": 3.635928005625166, + "tokens_seen": 1039138816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460481444332999, + "loss": 2.6605, + "theoretical_loss": 3.6359063384163397, + "tokens_seen": 1039204352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034603811434302914, + "loss": 2.6383, + "theoretical_loss": 3.635884672956452, + "tokens_seen": 1039269888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034602808425275827, + "loss": 2.7481, + "theoretical_loss": 3.6358630092452504, + "tokens_seen": 1039335424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460180541624875, + "loss": 2.5954, + "theoretical_loss": 3.6358413472824838, + "tokens_seen": 1039400960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034600802407221663, + "loss": 2.5512, + "theoretical_loss": 3.635819687067901, + "tokens_seen": 1039466496 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034599799398194587, + "loss": 2.6184, + "theoretical_loss": 3.6357980286012506, + "tokens_seen": 1039532032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034598796389167505, + "loss": 2.7418, + "theoretical_loss": 3.6357763718822813, + "tokens_seen": 1039597568 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034597793380140423, + "loss": 2.6507, + "theoretical_loss": 3.635754716910742, + "tokens_seen": 1039663104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003459679037111334, + "loss": 2.3988, + "theoretical_loss": 3.6357330636863816, + "tokens_seen": 1039728640 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034595787362086264, + "loss": 2.6205, + "theoretical_loss": 3.6357114122089493, + "tokens_seen": 1039794176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034594784353059177, + "loss": 2.858, + "theoretical_loss": 3.6356897624781936, + "tokens_seen": 1039859712 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345937813440321, + "loss": 2.6748, + "theoretical_loss": 3.635668114493863, + "tokens_seen": 1039925248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034592778335005013, + "loss": 2.5437, + "theoretical_loss": 3.635646468255708, + "tokens_seen": 1039990784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034591775325977937, + "loss": 2.4441, + "theoretical_loss": 3.635624823763477, + "tokens_seen": 1040056320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034590772316950855, + "loss": 2.562, + "theoretical_loss": 3.6356031810169185, + "tokens_seen": 1040121856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034589769307923773, + "loss": 2.7612, + "theoretical_loss": 3.635581540015783, + "tokens_seen": 1040187392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458876629889669, + "loss": 2.6531, + "theoretical_loss": 3.635559900759819, + "tokens_seen": 1040252928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458776328986961, + "loss": 2.8936, + "theoretical_loss": 3.635538263248776, + "tokens_seen": 1040318464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 580938, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.539257049560547, + "objective/train/theoretical_loss": 3.635516627482403, + "objective/train/tokens_used": 1060844000, + "theoretical_loss": 3.635516627482403, + "tokens_seen": 1040384000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458676028084253, + "loss": 2.5525, + "theoretical_loss": 3.635516627482403, + "tokens_seen": 1040384000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458575727181545, + "loss": 2.6589, + "theoretical_loss": 3.6354949934604504, + "tokens_seen": 1040449536 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034584754262788364, + "loss": 2.7115, + "theoretical_loss": 3.6354733611826666, + "tokens_seen": 1040515072 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034583751253761287, + "loss": 2.6772, + "theoretical_loss": 3.6354517306488017, + "tokens_seen": 1040580608 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345827482447342, + "loss": 2.6409, + "theoretical_loss": 3.6354301018586055, + "tokens_seen": 1040646144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034581745235707123, + "loss": 2.9847, + "theoretical_loss": 3.635408474811827, + "tokens_seen": 1040711680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458074222668004, + "loss": 2.5639, + "theoretical_loss": 3.635386849508217, + "tokens_seen": 1040777216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457973921765296, + "loss": 2.6658, + "theoretical_loss": 3.635365225947524, + "tokens_seen": 1040842752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457873620862588, + "loss": 2.5532, + "theoretical_loss": 3.635343604129498, + "tokens_seen": 1040908288 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345777331995988, + "loss": 2.7893, + "theoretical_loss": 3.63532198405389, + "tokens_seen": 1040973824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034576730190571714, + "loss": 2.6635, + "theoretical_loss": 3.6353003657204486, + "tokens_seen": 1041039360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457572718154464, + "loss": 2.5693, + "theoretical_loss": 3.635278749128924, + "tokens_seen": 1041104896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457472417251755, + "loss": 2.6342, + "theoretical_loss": 3.635257134279067, + "tokens_seen": 1041170432 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034573721163490474, + "loss": 2.6137, + "theoretical_loss": 3.6352355211706264, + "tokens_seen": 1041235968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457271815446339, + "loss": 2.7036, + "theoretical_loss": 3.635213909803354, + "tokens_seen": 1041301504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457171514543631, + "loss": 2.712, + "theoretical_loss": 3.635192300176998, + "tokens_seen": 1041367040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457071213640923, + "loss": 2.7841, + "theoretical_loss": 3.63517069229131, + "tokens_seen": 1041432576 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034569709127382146, + "loss": 2.7111, + "theoretical_loss": 3.63514908614604, + "tokens_seen": 1041498112 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034568706118355064, + "loss": 2.717, + "theoretical_loss": 3.6351274817409376, + "tokens_seen": 1041563648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456770310932799, + "loss": 2.4966, + "theoretical_loss": 3.6351058790757538, + "tokens_seen": 1041629184 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345667001003009, + "loss": 2.7355, + "theoretical_loss": 3.6350842781502393, + "tokens_seen": 1041694720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034565697091273824, + "loss": 2.691, + "theoretical_loss": 3.635062678964144, + "tokens_seen": 1041760256 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034564694082246737, + "loss": 2.6776, + "theoretical_loss": 3.635041081517219, + "tokens_seen": 1041825792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456369107321966, + "loss": 2.5323, + "theoretical_loss": 3.635019485809214, + "tokens_seen": 1041891328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456268806419258, + "loss": 2.701, + "theoretical_loss": 3.6349978918398804, + "tokens_seen": 1041956864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 581557, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8283419609069824, + "objective/train/theoretical_loss": 3.6349762996089683, + "objective/train/tokens_used": 1062482400, + "theoretical_loss": 3.6349762996089683, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034561685055165496, + "loss": 2.6366, + "theoretical_loss": 3.6349762996089683, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034560682046138415, + "loss": 2.694, + "theoretical_loss": 3.634954709116229, + "tokens_seen": 1042087936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455967903711134, + "loss": 2.6484, + "theoretical_loss": 3.6349331203614126, + "tokens_seen": 1042153472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455867602808425, + "loss": 2.6699, + "theoretical_loss": 3.6349115333442708, + "tokens_seen": 1042219008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034557673019057174, + "loss": 2.8731, + "theoretical_loss": 3.6348899480645542, + "tokens_seen": 1042284544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034556670010030087, + "loss": 2.5907, + "theoretical_loss": 3.634868364522013, + "tokens_seen": 1042350080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455566700100301, + "loss": 2.7887, + "theoretical_loss": 3.634846782716399, + "tokens_seen": 1042415616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455466399197593, + "loss": 2.58, + "theoretical_loss": 3.634825202647463, + "tokens_seen": 1042481152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034553660982948847, + "loss": 2.5548, + "theoretical_loss": 3.634803624314956, + "tokens_seen": 1042546688 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034552657973921765, + "loss": 2.6047, + "theoretical_loss": 3.634782047718629, + "tokens_seen": 1042612224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034551654964894683, + "loss": 2.6368, + "theoretical_loss": 3.6347604728582334, + "tokens_seen": 1042677760 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345506519558676, + "loss": 2.752, + "theoretical_loss": 3.6347388997335206, + "tokens_seen": 1042743296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034549648946840525, + "loss": 2.7275, + "theoretical_loss": 3.6347173283442418, + "tokens_seen": 1042808832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454864593781344, + "loss": 2.6721, + "theoretical_loss": 3.634695758690148, + "tokens_seen": 1042874368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454764292878636, + "loss": 2.7311, + "theoretical_loss": 3.6346741907709914, + "tokens_seen": 1042939904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034546639919759274, + "loss": 2.7933, + "theoretical_loss": 3.634652624586522, + "tokens_seen": 1043005440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034545636910732197, + "loss": 2.5669, + "theoretical_loss": 3.634631060136493, + "tokens_seen": 1043070976 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034544633901705115, + "loss": 2.6247, + "theoretical_loss": 3.6346094974206546, + "tokens_seen": 1043136512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034543630892678033, + "loss": 2.7361, + "theoretical_loss": 3.634587936438759, + "tokens_seen": 1043202048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454262788365095, + "loss": 2.385, + "theoretical_loss": 3.6345663771905583, + "tokens_seen": 1043267584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034541624874623875, + "loss": 2.6379, + "theoretical_loss": 3.6345448196758032, + "tokens_seen": 1043333120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454062186559679, + "loss": 2.7274, + "theoretical_loss": 3.634523263894246, + "tokens_seen": 1043398656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453961885656971, + "loss": 2.6841, + "theoretical_loss": 3.6345017098456385, + "tokens_seen": 1043464192 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034538615847542624, + "loss": 2.5927, + "theoretical_loss": 3.6344801575297323, + "tokens_seen": 1043529728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453761283851555, + "loss": 2.718, + "theoretical_loss": 3.63445860694628, + "tokens_seen": 1043595264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 582812, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.661972761154175, + "objective/train/theoretical_loss": 3.634437058095033, + "objective/train/tokens_used": 1064120800, + "theoretical_loss": 3.634437058095033, + "tokens_seen": 1043660800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034536609829488466, + "loss": 2.6098, + "theoretical_loss": 3.634437058095033, + "tokens_seen": 1043660800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034535606820461384, + "loss": 2.8238, + "theoretical_loss": 3.6344155109757432, + "tokens_seen": 1043726336 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345346038114343, + "loss": 2.8692, + "theoretical_loss": 3.6343939655881625, + "tokens_seen": 1043791872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453360080240722, + "loss": 2.5667, + "theoretical_loss": 3.634372421932044, + "tokens_seen": 1043857408 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034532597793380143, + "loss": 2.9155, + "theoretical_loss": 3.6343508800071387, + "tokens_seen": 1043922944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453159478435306, + "loss": 2.7324, + "theoretical_loss": 3.634329339813199, + "tokens_seen": 1043988480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453059177532598, + "loss": 2.6672, + "theoretical_loss": 3.634307801349978, + "tokens_seen": 1044054016 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345295887662989, + "loss": 2.6618, + "theoretical_loss": 3.6342862646172276, + "tokens_seen": 1044119552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452858575727182, + "loss": 2.7461, + "theoretical_loss": 3.6342647296147, + "tokens_seen": 1044185088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034527582748244734, + "loss": 2.7409, + "theoretical_loss": 3.6342431963421475, + "tokens_seen": 1044250624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452657973921766, + "loss": 2.7384, + "theoretical_loss": 3.634221664799323, + "tokens_seen": 1044316160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452557673019057, + "loss": 2.7435, + "theoretical_loss": 3.6342001349859783, + "tokens_seen": 1044381696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034524573721163494, + "loss": 2.7216, + "theoretical_loss": 3.634178606901867, + "tokens_seen": 1044447232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452357071213641, + "loss": 2.6133, + "theoretical_loss": 3.6341570805467405, + "tokens_seen": 1044512768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452256770310933, + "loss": 2.6797, + "theoretical_loss": 3.6341355559203525, + "tokens_seen": 1044578304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452156469408225, + "loss": 2.7254, + "theoretical_loss": 3.6341140330224553, + "tokens_seen": 1044643840 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034520561685055166, + "loss": 2.627, + "theoretical_loss": 3.634092511852802, + "tokens_seen": 1044709376 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034519558676028084, + "loss": 2.5409, + "theoretical_loss": 3.6340709924111447, + "tokens_seen": 1044774912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451855566700101, + "loss": 2.5336, + "theoretical_loss": 3.634049474697237, + "tokens_seen": 1044840448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451755265797392, + "loss": 2.7567, + "theoretical_loss": 3.6340279587108313, + "tokens_seen": 1044905984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034516549648946844, + "loss": 2.6346, + "theoretical_loss": 3.634006444451681, + "tokens_seen": 1044971520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034515546639919757, + "loss": 2.5416, + "theoretical_loss": 3.6339849319195388, + "tokens_seen": 1045037056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451454363089268, + "loss": 2.5917, + "theoretical_loss": 3.6339634211141583, + "tokens_seen": 1045102592 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345135406218656, + "loss": 2.7407, + "theoretical_loss": 3.633941912035292, + "tokens_seen": 1045168128 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034512537612838517, + "loss": 2.4952, + "theoretical_loss": 3.633920404682693, + "tokens_seen": 1045233664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 583395, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6361796855926514, + "objective/train/theoretical_loss": 3.633898899056115, + "objective/train/tokens_used": 1065759200, + "theoretical_loss": 3.633898899056115, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034511534603811435, + "loss": 2.7576, + "theoretical_loss": 3.633898899056115, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451053159478436, + "loss": 2.5862, + "theoretical_loss": 3.633877395155311, + "tokens_seen": 1045364736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450952858575727, + "loss": 2.631, + "theoretical_loss": 3.6338558929800353, + "tokens_seen": 1045430272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034508525576730194, + "loss": 2.8207, + "theoretical_loss": 3.63383439253004, + "tokens_seen": 1045495808 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034507522567703107, + "loss": 2.5697, + "theoretical_loss": 3.6338128938050787, + "tokens_seen": 1045561344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450651955867603, + "loss": 2.6425, + "theoretical_loss": 3.633791396804905, + "tokens_seen": 1045626880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450551654964895, + "loss": 2.7739, + "theoretical_loss": 3.6337699015292735, + "tokens_seen": 1045692416 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034504513540621867, + "loss": 2.3681, + "theoretical_loss": 3.633748407977936, + "tokens_seen": 1045757952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034503510531594785, + "loss": 2.8102, + "theoretical_loss": 3.6337269161506476, + "tokens_seen": 1045823488 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034502507522567703, + "loss": 2.4205, + "theoretical_loss": 3.633705426047161, + "tokens_seen": 1045889024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450150451354062, + "loss": 2.5667, + "theoretical_loss": 3.6336839376672305, + "tokens_seen": 1045954560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034500501504513545, + "loss": 2.4626, + "theoretical_loss": 3.63366245101061, + "tokens_seen": 1046020096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449949849548646, + "loss": 2.503, + "theoretical_loss": 3.633640966077053, + "tokens_seen": 1046085632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449849548645938, + "loss": 2.7242, + "theoretical_loss": 3.633619482866313, + "tokens_seen": 1046151168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034497492477432294, + "loss": 2.7701, + "theoretical_loss": 3.633598001378145, + "tokens_seen": 1046216704 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034496489468405217, + "loss": 2.6704, + "theoretical_loss": 3.633576521612302, + "tokens_seen": 1046282240 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034495486459378135, + "loss": 2.5448, + "theoretical_loss": 3.6335550435685384, + "tokens_seen": 1046347776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034494483450351053, + "loss": 2.559, + "theoretical_loss": 3.6335335672466087, + "tokens_seen": 1046413312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449348044132397, + "loss": 2.853, + "theoretical_loss": 3.6335120926462663, + "tokens_seen": 1046478848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034492477432296895, + "loss": 2.4466, + "theoretical_loss": 3.633490619767266, + "tokens_seen": 1046544384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449147442326981, + "loss": 2.609, + "theoretical_loss": 3.6334691486093615, + "tokens_seen": 1046609920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449047141424273, + "loss": 2.5306, + "theoretical_loss": 3.633447679172308, + "tokens_seen": 1046675456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034489468405215644, + "loss": 2.4962, + "theoretical_loss": 3.6334262114558586, + "tokens_seen": 1046740992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448846539618857, + "loss": 2.8402, + "theoretical_loss": 3.6334047454597687, + "tokens_seen": 1046806528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034487462387161486, + "loss": 2.4984, + "theoretical_loss": 3.633383281183792, + "tokens_seen": 1046872064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 584517, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.44215989112854, + "objective/train/theoretical_loss": 3.6333618186276837, + "objective/train/tokens_used": 1067397600, + "theoretical_loss": 3.6333618186276837, + "tokens_seen": 1046937600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034486459378134404, + "loss": 2.5829, + "theoretical_loss": 3.6333618186276837, + "tokens_seen": 1046937600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448545636910732, + "loss": 2.6031, + "theoretical_loss": 3.633340357791198, + "tokens_seen": 1047003136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448445336008024, + "loss": 2.5413, + "theoretical_loss": 3.6333188986740894, + "tokens_seen": 1047068672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448345035105316, + "loss": 2.5823, + "theoretical_loss": 3.6332974412761128, + "tokens_seen": 1047134208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448244734202608, + "loss": 2.5924, + "theoretical_loss": 3.6332759855970225, + "tokens_seen": 1047199744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034481444332998994, + "loss": 2.7455, + "theoretical_loss": 3.633254531636574, + "tokens_seen": 1047265280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448044132397192, + "loss": 2.4347, + "theoretical_loss": 3.633233079394521, + "tokens_seen": 1047330816 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034479438314944836, + "loss": 2.6795, + "theoretical_loss": 3.633211628870619, + "tokens_seen": 1047396352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034478435305917754, + "loss": 2.5025, + "theoretical_loss": 3.6331901800646236, + "tokens_seen": 1047461888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447743229689067, + "loss": 2.679, + "theoretical_loss": 3.633168732976288, + "tokens_seen": 1047527424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447642928786359, + "loss": 2.6558, + "theoretical_loss": 3.633147287605369, + "tokens_seen": 1047592960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447542627883651, + "loss": 2.6364, + "theoretical_loss": 3.6331258439516203, + "tokens_seen": 1047658496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447442326980943, + "loss": 2.4161, + "theoretical_loss": 3.633104402014798, + "tokens_seen": 1047724032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034473420260782345, + "loss": 2.5527, + "theoretical_loss": 3.633082961794656, + "tokens_seen": 1047789568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447241725175527, + "loss": 2.5928, + "theoretical_loss": 3.6330615232909507, + "tokens_seen": 1047855104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447141424272818, + "loss": 2.625, + "theoretical_loss": 3.6330400865034367, + "tokens_seen": 1047920640 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034470411233701104, + "loss": 2.6341, + "theoretical_loss": 3.63301865143187, + "tokens_seen": 1047986176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446940822467402, + "loss": 2.5154, + "theoretical_loss": 3.6329972180760057, + "tokens_seen": 1048051712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446840521564694, + "loss": 2.5054, + "theoretical_loss": 3.632975786435598, + "tokens_seen": 1048117248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446740220661986, + "loss": 2.7753, + "theoretical_loss": 3.632954356510404, + "tokens_seen": 1048182784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034466399197592777, + "loss": 2.8493, + "theoretical_loss": 3.6329329283001783, + "tokens_seen": 1048248320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034465396188565695, + "loss": 2.6918, + "theoretical_loss": 3.632911501804677, + "tokens_seen": 1048313856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446439317953862, + "loss": 2.6121, + "theoretical_loss": 3.632890077023655, + "tokens_seen": 1048379392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446339017051153, + "loss": 2.5529, + "theoretical_loss": 3.6328686539568684, + "tokens_seen": 1048444928 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034462387161484455, + "loss": 2.4928, + "theoretical_loss": 3.6328472326040737, + "tokens_seen": 1048510464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 585002, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9021494388580322, + "objective/train/theoretical_loss": 3.6328258129650246, + "objective/train/tokens_used": 1069036000, + "theoretical_loss": 3.6328258129650246, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034461384152457373, + "loss": 2.5486, + "theoretical_loss": 3.6328258129650246, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446038114343029, + "loss": 2.7026, + "theoretical_loss": 3.6328043950394786, + "tokens_seen": 1048641536 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034459378134403214, + "loss": 2.4971, + "theoretical_loss": 3.632782978827191, + "tokens_seen": 1048707072 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034458375125376127, + "loss": 2.5622, + "theoretical_loss": 3.6327615643279176, + "tokens_seen": 1048772608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445737211634905, + "loss": 2.7683, + "theoretical_loss": 3.6327401515414146, + "tokens_seen": 1048838144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445636910732197, + "loss": 2.612, + "theoretical_loss": 3.632718740467438, + "tokens_seen": 1048903680 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034455366098294887, + "loss": 2.8323, + "theoretical_loss": 3.6326973311057436, + "tokens_seen": 1048969216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034454363089267805, + "loss": 2.6183, + "theoretical_loss": 3.6326759234560875, + "tokens_seen": 1049034752 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034453360080240723, + "loss": 2.6601, + "theoretical_loss": 3.632654517518226, + "tokens_seen": 1049100288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445235707121364, + "loss": 2.8211, + "theoretical_loss": 3.6326331132919156, + "tokens_seen": 1049165824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034451354062186565, + "loss": 2.6244, + "theoretical_loss": 3.632611710776912, + "tokens_seen": 1049231360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445035105315948, + "loss": 2.4925, + "theoretical_loss": 3.632590309972972, + "tokens_seen": 1049296896 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344493480441324, + "loss": 2.9594, + "theoretical_loss": 3.6325689108798516, + "tokens_seen": 1049362432 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034448345035105314, + "loss": 2.7034, + "theoretical_loss": 3.6325475134973066, + "tokens_seen": 1049427968 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034447342026078237, + "loss": 2.5489, + "theoretical_loss": 3.632526117825095, + "tokens_seen": 1049493504 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034446339017051155, + "loss": 2.6613, + "theoretical_loss": 3.632504723862972, + "tokens_seen": 1049559040 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034445336008024073, + "loss": 2.3941, + "theoretical_loss": 3.632483331610695, + "tokens_seen": 1049624576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444433299899699, + "loss": 2.6026, + "theoretical_loss": 3.63246194106802, + "tokens_seen": 1049690112 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034443329989969915, + "loss": 2.8111, + "theoretical_loss": 3.632440552234704, + "tokens_seen": 1049755648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444232698094283, + "loss": 2.7581, + "theoretical_loss": 3.6324191651105036, + "tokens_seen": 1049821184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444132397191575, + "loss": 2.5065, + "theoretical_loss": 3.632397779695175, + "tokens_seen": 1049886720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034440320962888664, + "loss": 2.4674, + "theoretical_loss": 3.632376395988476, + "tokens_seen": 1049952256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443931795386159, + "loss": 2.3677, + "theoretical_loss": 3.632355013990163, + "tokens_seen": 1050017792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034438314944834506, + "loss": 2.6539, + "theoretical_loss": 3.632333633699992, + "tokens_seen": 1050083328 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034437311935807424, + "loss": 2.3852, + "theoretical_loss": 3.6323122551177214, + "tokens_seen": 1050148864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 586227, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.932581663131714, + "objective/train/theoretical_loss": 3.632290878243108, + "objective/train/tokens_used": 1070674400, + "theoretical_loss": 3.632290878243108, + "tokens_seen": 1050214400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443630892678034, + "loss": 2.6061, + "theoretical_loss": 3.632290878243108, + "tokens_seen": 1050214400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443530591775326, + "loss": 2.6796, + "theoretical_loss": 3.632269503075908, + "tokens_seen": 1050279936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443430290872618, + "loss": 2.7109, + "theoretical_loss": 3.6322481296158786, + "tokens_seen": 1050345472 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344332998996991, + "loss": 2.5937, + "theoretical_loss": 3.632226757862778, + "tokens_seen": 1050411008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034432296890672014, + "loss": 2.7738, + "theoretical_loss": 3.632205387816362, + "tokens_seen": 1050476544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443129388164494, + "loss": 2.7724, + "theoretical_loss": 3.632184019476389, + "tokens_seen": 1050542080 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034430290872617856, + "loss": 2.635, + "theoretical_loss": 3.6321626528426156, + "tokens_seen": 1050607616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034429287863590774, + "loss": 2.5516, + "theoretical_loss": 3.6321412879147994, + "tokens_seen": 1050673152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442828485456369, + "loss": 2.7096, + "theoretical_loss": 3.632119924692698, + "tokens_seen": 1050738688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442728184553661, + "loss": 2.5264, + "theoretical_loss": 3.632098563176068, + "tokens_seen": 1050804224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442627883650953, + "loss": 2.4598, + "theoretical_loss": 3.632077203364668, + "tokens_seen": 1050869760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442527582748245, + "loss": 2.4605, + "theoretical_loss": 3.632055845258255, + "tokens_seen": 1050935296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034424272818455365, + "loss": 2.6781, + "theoretical_loss": 3.6320344888565863, + "tokens_seen": 1051000832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442326980942829, + "loss": 2.6028, + "theoretical_loss": 3.6320131341594206, + "tokens_seen": 1051066368 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344222668004012, + "loss": 2.5341, + "theoretical_loss": 3.6319917811665143, + "tokens_seen": 1051131904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034421263791374124, + "loss": 2.5869, + "theoretical_loss": 3.631970429877626, + "tokens_seen": 1051197440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442026078234704, + "loss": 2.7181, + "theoretical_loss": 3.6319490802925136, + "tokens_seen": 1051262976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441925777331996, + "loss": 2.8044, + "theoretical_loss": 3.631927732410934, + "tokens_seen": 1051328512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441825476429288, + "loss": 2.6143, + "theoretical_loss": 3.6319063862326457, + "tokens_seen": 1051394048 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034417251755265797, + "loss": 2.8513, + "theoretical_loss": 3.631885041757407, + "tokens_seen": 1051459584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034416248746238715, + "loss": 2.5423, + "theoretical_loss": 3.631863698984975, + "tokens_seen": 1051525120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441524573721164, + "loss": 2.5051, + "theoretical_loss": 3.631842357915108, + "tokens_seen": 1051590656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441424272818455, + "loss": 2.4431, + "theoretical_loss": 3.631821018547565, + "tokens_seen": 1051656192 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034413239719157475, + "loss": 2.6653, + "theoretical_loss": 3.6317996808821027, + "tokens_seen": 1051721728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034412236710130393, + "loss": 2.5794, + "theoretical_loss": 3.631778344918481, + "tokens_seen": 1051787264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 587166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6821694374084473, + "objective/train/theoretical_loss": 3.6317570106564565, + "objective/train/tokens_used": 1072312800, + "theoretical_loss": 3.6317570106564565, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441123370110331, + "loss": 2.6543, + "theoretical_loss": 3.6317570106564565, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441023069207623, + "loss": 2.6049, + "theoretical_loss": 3.6317356780957883, + "tokens_seen": 1051918336 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034409227683049147, + "loss": 2.2439, + "theoretical_loss": 3.6317143472362345, + "tokens_seen": 1051983872 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034408224674022065, + "loss": 2.7155, + "theoretical_loss": 3.631693018077554, + "tokens_seen": 1052049408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440722166499499, + "loss": 2.5276, + "theoretical_loss": 3.6316716906195046, + "tokens_seen": 1052114944 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344062186559679, + "loss": 2.512, + "theoretical_loss": 3.6316503648618452, + "tokens_seen": 1052180480 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034405215646940825, + "loss": 2.7207, + "theoretical_loss": 3.6316290408043335, + "tokens_seen": 1052246016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440421263791374, + "loss": 2.6449, + "theoretical_loss": 3.6316077184467295, + "tokens_seen": 1052311552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440320962888666, + "loss": 2.6526, + "theoretical_loss": 3.6315863977887908, + "tokens_seen": 1052377088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440220661985958, + "loss": 2.6497, + "theoretical_loss": 3.6315650788302762, + "tokens_seen": 1052442624 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344012036108325, + "loss": 2.7508, + "theoretical_loss": 3.6315437615709447, + "tokens_seen": 1052508160 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034400200601805416, + "loss": 2.8709, + "theoretical_loss": 3.631522446010555, + "tokens_seen": 1052573696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034399197592778334, + "loss": 2.565, + "theoretical_loss": 3.6315011321488657, + "tokens_seen": 1052639232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439819458375125, + "loss": 2.6508, + "theoretical_loss": 3.631479819985636, + "tokens_seen": 1052704768 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034397191574724175, + "loss": 2.7176, + "theoretical_loss": 3.6314585095206247, + "tokens_seen": 1052770304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439618856569709, + "loss": 2.7527, + "theoretical_loss": 3.6314372007535907, + "tokens_seen": 1052835840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439518555667001, + "loss": 2.631, + "theoretical_loss": 3.6314158936842933, + "tokens_seen": 1052901376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439418254764293, + "loss": 2.7249, + "theoretical_loss": 3.6313945883124914, + "tokens_seen": 1052966912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439317953861585, + "loss": 2.6789, + "theoretical_loss": 3.6313732846379434, + "tokens_seen": 1053032448 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034392176529588766, + "loss": 2.8109, + "theoretical_loss": 3.63135198266041, + "tokens_seen": 1053097984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034391173520561684, + "loss": 2.4002, + "theoretical_loss": 3.631330682379649, + "tokens_seen": 1053163520 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343901705115346, + "loss": 2.6808, + "theoretical_loss": 3.631309383795421, + "tokens_seen": 1053229056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034389167502507526, + "loss": 2.5586, + "theoretical_loss": 3.631288086907484, + "tokens_seen": 1053294592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438816449348044, + "loss": 2.6458, + "theoretical_loss": 3.631266791715598, + "tokens_seen": 1053360128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438716148445336, + "loss": 2.9422, + "theoretical_loss": 3.6312454982195224, + "tokens_seen": 1053425664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 587595, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.472946882247925, + "objective/train/theoretical_loss": 3.631224206419017, + "objective/train/tokens_used": 1073951200, + "theoretical_loss": 3.631224206419017, + "tokens_seen": 1053491200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034386158475426275, + "loss": 2.5474, + "theoretical_loss": 3.631224206419017, + "tokens_seen": 1053491200 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343851554663992, + "loss": 2.5543, + "theoretical_loss": 3.631202916313841, + "tokens_seen": 1053556736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438415245737212, + "loss": 2.6514, + "theoretical_loss": 3.6311816279037537, + "tokens_seen": 1053622272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034383149448345034, + "loss": 2.71, + "theoretical_loss": 3.6311603411885147, + "tokens_seen": 1053687808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438214643931796, + "loss": 2.6042, + "theoretical_loss": 3.631139056167884, + "tokens_seen": 1053753344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034381143430290876, + "loss": 2.5687, + "theoretical_loss": 3.6311177728416215, + "tokens_seen": 1053818880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034380140421263794, + "loss": 2.7691, + "theoretical_loss": 3.6310964912094867, + "tokens_seen": 1053884416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437913741223671, + "loss": 2.7997, + "theoretical_loss": 3.6310752112712392, + "tokens_seen": 1053949952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437813440320963, + "loss": 2.4598, + "theoretical_loss": 3.631053933026639, + "tokens_seen": 1054015488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437713139418255, + "loss": 2.7018, + "theoretical_loss": 3.6310326564754467, + "tokens_seen": 1054081024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437612838515547, + "loss": 2.596, + "theoretical_loss": 3.631011381617421, + "tokens_seen": 1054146560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034375125376128385, + "loss": 2.9412, + "theoretical_loss": 3.630990108452323, + "tokens_seen": 1054212096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437412236710131, + "loss": 2.6816, + "theoretical_loss": 3.630968836979912, + "tokens_seen": 1054277632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437311935807422, + "loss": 2.6108, + "theoretical_loss": 3.6309475671999483, + "tokens_seen": 1054343168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034372116349047144, + "loss": 2.6602, + "theoretical_loss": 3.6309262991121924, + "tokens_seen": 1054408704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437111334002006, + "loss": 2.4562, + "theoretical_loss": 3.630905032716404, + "tokens_seen": 1054474240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437011033099298, + "loss": 2.6896, + "theoretical_loss": 3.630883768012344, + "tokens_seen": 1054539776 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343691073219659, + "loss": 2.8709, + "theoretical_loss": 3.6308625049997723, + "tokens_seen": 1054605312 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034368104312938817, + "loss": 2.5781, + "theoretical_loss": 3.630841243678449, + "tokens_seen": 1054670848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034367101303911735, + "loss": 2.7216, + "theoretical_loss": 3.630819984048135, + "tokens_seen": 1054736384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436609829488466, + "loss": 2.6508, + "theoretical_loss": 3.6307987261085906, + "tokens_seen": 1054801920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436509528585757, + "loss": 2.6724, + "theoretical_loss": 3.630777469859576, + "tokens_seen": 1054867456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034364092276830495, + "loss": 2.6541, + "theoretical_loss": 3.6307562153008517, + "tokens_seen": 1054932992 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034363089267803413, + "loss": 2.901, + "theoretical_loss": 3.6307349624321787, + "tokens_seen": 1054998528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436208625877633, + "loss": 2.5115, + "theoretical_loss": 3.630713711253318, + "tokens_seen": 1055064064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 588937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4390525817871094, + "objective/train/theoretical_loss": 3.6306924617640295, + "objective/train/tokens_used": 1075589600, + "theoretical_loss": 3.6306924617640295, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436108324974925, + "loss": 2.5418, + "theoretical_loss": 3.6306924617640295, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034360080240722167, + "loss": 2.7519, + "theoretical_loss": 3.6306712139640744, + "tokens_seen": 1055195136 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034359077231695085, + "loss": 2.6733, + "theoretical_loss": 3.6306499678532127, + "tokens_seen": 1055260672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435807422266801, + "loss": 2.5918, + "theoretical_loss": 3.6306287234312062, + "tokens_seen": 1055326208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435707121364092, + "loss": 2.5974, + "theoretical_loss": 3.630607480697816, + "tokens_seen": 1055391744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034356068204613845, + "loss": 2.6814, + "theoretical_loss": 3.6305862396528017, + "tokens_seen": 1055457280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435506519558676, + "loss": 2.6467, + "theoretical_loss": 3.6305650002959258, + "tokens_seen": 1055522816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435406218655968, + "loss": 2.5512, + "theoretical_loss": 3.6305437626269486, + "tokens_seen": 1055588352 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343530591775326, + "loss": 2.6324, + "theoretical_loss": 3.6305225266456302, + "tokens_seen": 1055653888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435205616850552, + "loss": 2.6879, + "theoretical_loss": 3.6305012923517337, + "tokens_seen": 1055719424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034351053159478436, + "loss": 2.3758, + "theoretical_loss": 3.630480059745019, + "tokens_seen": 1055784960 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034350050150451354, + "loss": 2.7239, + "theoretical_loss": 3.6304588288252475, + "tokens_seen": 1055850496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434904714142427, + "loss": 2.6559, + "theoretical_loss": 3.630437599592181, + "tokens_seen": 1055916032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034348044132397195, + "loss": 2.4738, + "theoretical_loss": 3.6304163720455795, + "tokens_seen": 1055981568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434704112337011, + "loss": 2.7831, + "theoretical_loss": 3.630395146185206, + "tokens_seen": 1056047104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434603811434303, + "loss": 2.6243, + "theoretical_loss": 3.630373922010821, + "tokens_seen": 1056112640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434503510531595, + "loss": 2.6946, + "theoretical_loss": 3.6303526995221866, + "tokens_seen": 1056178176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434403209628887, + "loss": 2.6034, + "theoretical_loss": 3.6303314787190635, + "tokens_seen": 1056243712 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034343029087261786, + "loss": 2.5507, + "theoretical_loss": 3.630310259601214, + "tokens_seen": 1056309248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034342026078234704, + "loss": 2.5951, + "theoretical_loss": 3.630289042168399, + "tokens_seen": 1056374784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434102306920762, + "loss": 2.7226, + "theoretical_loss": 3.630267826420381, + "tokens_seen": 1056440320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034340020060180546, + "loss": 2.7196, + "theoretical_loss": 3.630246612356921, + "tokens_seen": 1056505856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433901705115346, + "loss": 2.6928, + "theoretical_loss": 3.630225399977781, + "tokens_seen": 1056571392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433801404212638, + "loss": 2.5232, + "theoretical_loss": 3.630204189282723, + "tokens_seen": 1056636928 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034337011033099295, + "loss": 2.7453, + "theoretical_loss": 3.6301829802715084, + "tokens_seen": 1056702464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 589659, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.753281354904175, + "objective/train/theoretical_loss": 3.6301617729438993, + "objective/train/tokens_used": 1077228000, + "theoretical_loss": 3.6301617729438993, + "tokens_seen": 1056768000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433600802407222, + "loss": 2.8091, + "theoretical_loss": 3.6301617729438993, + "tokens_seen": 1056768000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034335005015045136, + "loss": 2.6981, + "theoretical_loss": 3.6301405672996583, + "tokens_seen": 1056833536 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034334002006018054, + "loss": 2.6649, + "theoretical_loss": 3.630119363338547, + "tokens_seen": 1056899072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433299899699097, + "loss": 2.6571, + "theoretical_loss": 3.6300981610603267, + "tokens_seen": 1056964608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433199598796389, + "loss": 2.6702, + "theoretical_loss": 3.6300769604647605, + "tokens_seen": 1057030144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433099297893681, + "loss": 2.8513, + "theoretical_loss": 3.63005576155161, + "tokens_seen": 1057095680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432998996990973, + "loss": 2.744, + "theoretical_loss": 3.6300345643206375, + "tokens_seen": 1057161216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034328986960882645, + "loss": 2.4055, + "theoretical_loss": 3.6300133687716056, + "tokens_seen": 1057226752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432798395185557, + "loss": 2.6993, + "theoretical_loss": 3.6299921749042765, + "tokens_seen": 1057292288 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034326980942828487, + "loss": 2.3819, + "theoretical_loss": 3.6299709827184117, + "tokens_seen": 1057357824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034325977933801405, + "loss": 2.8359, + "theoretical_loss": 3.6299497922137745, + "tokens_seen": 1057423360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034324974924774323, + "loss": 2.6268, + "theoretical_loss": 3.6299286033901272, + "tokens_seen": 1057488896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432397191574724, + "loss": 2.7105, + "theoretical_loss": 3.6299074162472325, + "tokens_seen": 1057554432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432296890672016, + "loss": 2.4748, + "theoretical_loss": 3.6298862307848516, + "tokens_seen": 1057619968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432196589769308, + "loss": 2.8688, + "theoretical_loss": 3.629865047002749, + "tokens_seen": 1057685504 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034320962888665995, + "loss": 2.6851, + "theoretical_loss": 3.6298438649006863, + "tokens_seen": 1057751040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431995987963892, + "loss": 2.7079, + "theoretical_loss": 3.629822684478426, + "tokens_seen": 1057816576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431895687061183, + "loss": 2.8602, + "theoretical_loss": 3.629801505735731, + "tokens_seen": 1057882112 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034317953861584755, + "loss": 2.5481, + "theoretical_loss": 3.629780328672364, + "tokens_seen": 1057947648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034316950852557673, + "loss": 2.8164, + "theoretical_loss": 3.629759153288089, + "tokens_seen": 1058013184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431594784353059, + "loss": 2.6221, + "theoretical_loss": 3.6297379795826674, + "tokens_seen": 1058078720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431494483450351, + "loss": 2.563, + "theoretical_loss": 3.629716807555863, + "tokens_seen": 1058144256 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034313941825476433, + "loss": 2.6197, + "theoretical_loss": 3.629695637207438, + "tokens_seen": 1058209792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034312938816449345, + "loss": 2.6229, + "theoretical_loss": 3.6296744685371554, + "tokens_seen": 1058275328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431193580742227, + "loss": 2.4343, + "theoretical_loss": 3.629653301544779, + "tokens_seen": 1058340864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 591033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.53261661529541, + "objective/train/theoretical_loss": 3.6296321362300716, + "objective/train/tokens_used": 1078866400, + "theoretical_loss": 3.6296321362300716, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431093279839518, + "loss": 2.5809, + "theoretical_loss": 3.6296321362300716, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034309929789368105, + "loss": 2.5506, + "theoretical_loss": 3.629610972592797, + "tokens_seen": 1058471936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430892678034103, + "loss": 2.8012, + "theoretical_loss": 3.6295898106327167, + "tokens_seen": 1058537472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430792377131394, + "loss": 2.7142, + "theoretical_loss": 3.629568650349596, + "tokens_seen": 1058603008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034306920762286865, + "loss": 2.7891, + "theoretical_loss": 3.6295474917431965, + "tokens_seen": 1058668544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430591775325978, + "loss": 2.7441, + "theoretical_loss": 3.6295263348132822, + "tokens_seen": 1058734080 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343049147442327, + "loss": 2.3854, + "theoretical_loss": 3.629505179559617, + "tokens_seen": 1058799616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430391173520562, + "loss": 2.6415, + "theoretical_loss": 3.6294840259819643, + "tokens_seen": 1058865152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430290872617854, + "loss": 2.8416, + "theoretical_loss": 3.6294628740800867, + "tokens_seen": 1058930688 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034301905717151456, + "loss": 2.6718, + "theoretical_loss": 3.6294417238537484, + "tokens_seen": 1058996224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034300902708124374, + "loss": 2.5243, + "theoretical_loss": 3.6294205753027127, + "tokens_seen": 1059061760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429989969909729, + "loss": 2.6848, + "theoretical_loss": 3.6293994284267437, + "tokens_seen": 1059127296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034298896690070215, + "loss": 2.7442, + "theoretical_loss": 3.6293782832256047, + "tokens_seen": 1059192832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429789368104313, + "loss": 2.6438, + "theoretical_loss": 3.629357139699059, + "tokens_seen": 1059258368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429689067201605, + "loss": 2.7197, + "theoretical_loss": 3.629335997846872, + "tokens_seen": 1059323904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429588766298897, + "loss": 2.4123, + "theoretical_loss": 3.6293148576688057, + "tokens_seen": 1059389440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429488465396189, + "loss": 2.662, + "theoretical_loss": 3.6292937191646253, + "tokens_seen": 1059454976 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034293881644934806, + "loss": 2.433, + "theoretical_loss": 3.6292725823340932, + "tokens_seen": 1059520512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034292878635907724, + "loss": 2.5597, + "theoretical_loss": 3.6292514471769755, + "tokens_seen": 1059586048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429187562688064, + "loss": 2.9441, + "theoretical_loss": 3.629230313693035, + "tokens_seen": 1059651584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034290872617853566, + "loss": 2.3666, + "theoretical_loss": 3.629209181882035, + "tokens_seen": 1059717120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428986960882648, + "loss": 2.9156, + "theoretical_loss": 3.629188051743741, + "tokens_seen": 1059782656 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342888665997994, + "loss": 2.6805, + "theoretical_loss": 3.6291669232779165, + "tokens_seen": 1059848192 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034287863590772315, + "loss": 2.5692, + "theoretical_loss": 3.629145796484326, + "tokens_seen": 1059913728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428686058174524, + "loss": 2.7361, + "theoretical_loss": 3.6291246713627334, + "tokens_seen": 1059979264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 592062, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5544235706329346, + "objective/train/theoretical_loss": 3.6291035479129032, + "objective/train/tokens_used": 1080504800, + "theoretical_loss": 3.6291035479129032, + "tokens_seen": 1060044800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034285857572718156, + "loss": 2.4871, + "theoretical_loss": 3.6291035479129032, + "tokens_seen": 1060044800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034284854563691074, + "loss": 2.3404, + "theoretical_loss": 3.6290824261346, + "tokens_seen": 1060110336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428385155466399, + "loss": 2.8714, + "theoretical_loss": 3.6290613060275883, + "tokens_seen": 1060175872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428284854563691, + "loss": 2.7667, + "theoretical_loss": 3.629040187591632, + "tokens_seen": 1060241408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428184553660983, + "loss": 2.7177, + "theoretical_loss": 3.6290190708264958, + "tokens_seen": 1060306944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428084252758275, + "loss": 2.7029, + "theoretical_loss": 3.6289979557319443, + "tokens_seen": 1060372480 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034279839518555665, + "loss": 2.634, + "theoretical_loss": 3.628976842307742, + "tokens_seen": 1060438016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427883650952859, + "loss": 2.5498, + "theoretical_loss": 3.6289557305536544, + "tokens_seen": 1060503552 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034277833500501507, + "loss": 2.6537, + "theoretical_loss": 3.628934620469445, + "tokens_seen": 1060569088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034276830491474425, + "loss": 2.7612, + "theoretical_loss": 3.6289135120548783, + "tokens_seen": 1060634624 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034275827482447343, + "loss": 2.845, + "theoretical_loss": 3.6288924053097205, + "tokens_seen": 1060700160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427482447342026, + "loss": 2.8157, + "theoretical_loss": 3.628871300233736, + "tokens_seen": 1060765696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427382146439318, + "loss": 2.9871, + "theoretical_loss": 3.628850196826689, + "tokens_seen": 1060831232 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342728184553661, + "loss": 3.0334, + "theoretical_loss": 3.628829095088345, + "tokens_seen": 1060896768 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034271815446339015, + "loss": 2.5297, + "theoretical_loss": 3.6288079950184686, + "tokens_seen": 1060962304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427081243731194, + "loss": 2.5689, + "theoretical_loss": 3.628786896616825, + "tokens_seen": 1061027840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426980942828485, + "loss": 2.7144, + "theoretical_loss": 3.6287657998831797, + "tokens_seen": 1061093376 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034268806419257775, + "loss": 2.6815, + "theoretical_loss": 3.628744704817297, + "tokens_seen": 1061158912 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034267803410230693, + "loss": 2.7748, + "theoretical_loss": 3.628723611418943, + "tokens_seen": 1061224448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426680040120361, + "loss": 2.6297, + "theoretical_loss": 3.6287025196878826, + "tokens_seen": 1061289984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426579739217653, + "loss": 2.5459, + "theoretical_loss": 3.6286814296238803, + "tokens_seen": 1061355520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034264794383149453, + "loss": 2.872, + "theoretical_loss": 3.6286603412267024, + "tokens_seen": 1061421056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034263791374122366, + "loss": 2.6676, + "theoretical_loss": 3.6286392544961137, + "tokens_seen": 1061486592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426278836509529, + "loss": 2.6477, + "theoretical_loss": 3.62861816943188, + "tokens_seen": 1061552128 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342617853560682, + "loss": 2.8712, + "theoretical_loss": 3.6285970860337664, + "tokens_seen": 1061617664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 592448, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1389501094818115, + "objective/train/theoretical_loss": 3.6285760043015385, + "objective/train/tokens_used": 1082143200, + "theoretical_loss": 3.6285760043015385, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034260782347041125, + "loss": 2.7504, + "theoretical_loss": 3.6285760043015385, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034259779338014043, + "loss": 2.6796, + "theoretical_loss": 3.6285549242349626, + "tokens_seen": 1061748736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425877632898696, + "loss": 2.6368, + "theoretical_loss": 3.628533845833803, + "tokens_seen": 1061814272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425777331995988, + "loss": 2.8308, + "theoretical_loss": 3.628512769097826, + "tokens_seen": 1061879808 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342567703109328, + "loss": 2.613, + "theoretical_loss": 3.6284916940267973, + "tokens_seen": 1061945344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034255767301905716, + "loss": 2.9996, + "theoretical_loss": 3.628470620620482, + "tokens_seen": 1062010880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425476429287864, + "loss": 2.6594, + "theoretical_loss": 3.6284495488786472, + "tokens_seen": 1062076416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425376128385155, + "loss": 2.523, + "theoretical_loss": 3.628428478801058, + "tokens_seen": 1062141952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034252758274824476, + "loss": 2.87, + "theoretical_loss": 3.62840741038748, + "tokens_seen": 1062207488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425175526579739, + "loss": 2.608, + "theoretical_loss": 3.62838634363768, + "tokens_seen": 1062273024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425075225677031, + "loss": 2.6211, + "theoretical_loss": 3.628365278551423, + "tokens_seen": 1062338560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424974924774323, + "loss": 2.8357, + "theoretical_loss": 3.6283442151284753, + "tokens_seen": 1062404096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424874623871615, + "loss": 2.7023, + "theoretical_loss": 3.628323153368603, + "tokens_seen": 1062469632 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034247743229689066, + "loss": 2.2979, + "theoretical_loss": 3.628302093271573, + "tokens_seen": 1062535168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424674022066199, + "loss": 2.5555, + "theoretical_loss": 3.6282810348371504, + "tokens_seen": 1062600704 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342457372116349, + "loss": 2.4632, + "theoretical_loss": 3.6282599780651017, + "tokens_seen": 1062666240 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034244734202607826, + "loss": 2.9378, + "theoretical_loss": 3.6282389229551932, + "tokens_seen": 1062731776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424373119358074, + "loss": 2.6977, + "theoretical_loss": 3.6282178695071914, + "tokens_seen": 1062797312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424272818455366, + "loss": 2.4779, + "theoretical_loss": 3.6281968177208626, + "tokens_seen": 1062862848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424172517552658, + "loss": 2.7706, + "theoretical_loss": 3.628175767595973, + "tokens_seen": 1062928384 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342407221664995, + "loss": 2.7399, + "theoretical_loss": 3.628154719132289, + "tokens_seen": 1062993920 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034239719157472416, + "loss": 2.4667, + "theoretical_loss": 3.628133672329578, + "tokens_seen": 1063059456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034238716148445335, + "loss": 2.7334, + "theoretical_loss": 3.628112627187605, + "tokens_seen": 1063124992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423771313941825, + "loss": 2.8203, + "theoretical_loss": 3.628091583706138, + "tokens_seen": 1063190528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034236710130391176, + "loss": 2.614, + "theoretical_loss": 3.6280705418849433, + "tokens_seen": 1063256064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 593762, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.574810743331909, + "objective/train/theoretical_loss": 3.628049501723787, + "objective/train/tokens_used": 1083781600, + "theoretical_loss": 3.628049501723787, + "tokens_seen": 1063321600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423570712136409, + "loss": 2.7115, + "theoretical_loss": 3.628049501723787, + "tokens_seen": 1063321600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423470411233701, + "loss": 2.7239, + "theoretical_loss": 3.628028463222436, + "tokens_seen": 1063387136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423370110330993, + "loss": 2.5938, + "theoretical_loss": 3.628007426380657, + "tokens_seen": 1063452672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423269809428285, + "loss": 2.7494, + "theoretical_loss": 3.6279863911982178, + "tokens_seen": 1063518208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423169508525577, + "loss": 2.5914, + "theoretical_loss": 3.627965357674884, + "tokens_seen": 1063583744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034230692076228685, + "loss": 2.6498, + "theoretical_loss": 3.6279443258104234, + "tokens_seen": 1063649280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422968906720161, + "loss": 2.9348, + "theoretical_loss": 3.627923295604603, + "tokens_seen": 1063714816 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034228686058174527, + "loss": 2.6254, + "theoretical_loss": 3.6279022670571885, + "tokens_seen": 1063780352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034227683049147445, + "loss": 2.7505, + "theoretical_loss": 3.627881240167949, + "tokens_seen": 1063845888 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034226680040120363, + "loss": 2.6287, + "theoretical_loss": 3.62786021493665, + "tokens_seen": 1063911424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422567703109328, + "loss": 2.6029, + "theoretical_loss": 3.6278391913630594, + "tokens_seen": 1063976960 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342246740220662, + "loss": 2.8587, + "theoretical_loss": 3.627818169446944, + "tokens_seen": 1064042496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422367101303912, + "loss": 2.7094, + "theoretical_loss": 3.6277971491880714, + "tokens_seen": 1064108032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034222668004012035, + "loss": 2.5434, + "theoretical_loss": 3.627776130586209, + "tokens_seen": 1064173568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422166499498496, + "loss": 2.9045, + "theoretical_loss": 3.6277551136411237, + "tokens_seen": 1064239104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422066198595787, + "loss": 2.8314, + "theoretical_loss": 3.6277340983525828, + "tokens_seen": 1064304640 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034219658976930795, + "loss": 2.473, + "theoretical_loss": 3.6277130847203543, + "tokens_seen": 1064370176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034218655967903713, + "loss": 2.6314, + "theoretical_loss": 3.627692072744206, + "tokens_seen": 1064435712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421765295887663, + "loss": 2.4801, + "theoretical_loss": 3.6276710624239037, + "tokens_seen": 1064501248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421664994984955, + "loss": 2.6896, + "theoretical_loss": 3.627650053759217, + "tokens_seen": 1064566784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034215646940822473, + "loss": 2.8167, + "theoretical_loss": 3.6276290467499126, + "tokens_seen": 1064632320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034214643931795386, + "loss": 2.6443, + "theoretical_loss": 3.6276080413957583, + "tokens_seen": 1064697856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421364092276831, + "loss": 2.8321, + "theoretical_loss": 3.6275870376965216, + "tokens_seen": 1064763392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421263791374122, + "loss": 2.511, + "theoretical_loss": 3.62756603565197, + "tokens_seen": 1064828928 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034211634904714145, + "loss": 2.6694, + "theoretical_loss": 3.6275450352618726, + "tokens_seen": 1064894464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 594388, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.325084686279297, + "objective/train/theoretical_loss": 3.6275240365259958, + "objective/train/tokens_used": 1085420000, + "theoretical_loss": 3.6275240365259958, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034210631895687063, + "loss": 2.5833, + "theoretical_loss": 3.6275240365259958, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420962888665998, + "loss": 2.7116, + "theoretical_loss": 3.6275030394441083, + "tokens_seen": 1065025536 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342086258776329, + "loss": 2.5262, + "theoretical_loss": 3.627482044015978, + "tokens_seen": 1065091072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420762286860582, + "loss": 2.6584, + "theoretical_loss": 3.627461050241373, + "tokens_seen": 1065156608 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034206619859578736, + "loss": 2.6601, + "theoretical_loss": 3.6274400581200608, + "tokens_seen": 1065222144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420561685055166, + "loss": 2.6521, + "theoretical_loss": 3.62741906765181, + "tokens_seen": 1065287680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420461384152457, + "loss": 2.5302, + "theoretical_loss": 3.6273980788363884, + "tokens_seen": 1065353216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034203610832497496, + "loss": 2.4866, + "theoretical_loss": 3.6273770916735644, + "tokens_seen": 1065418752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420260782347041, + "loss": 2.614, + "theoretical_loss": 3.6273561061631066, + "tokens_seen": 1065484288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420160481444333, + "loss": 2.713, + "theoretical_loss": 3.6273351223047823, + "tokens_seen": 1065549824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420060180541625, + "loss": 2.6868, + "theoretical_loss": 3.6273141400983606, + "tokens_seen": 1065615360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419959879638917, + "loss": 2.4458, + "theoretical_loss": 3.6272931595436098, + "tokens_seen": 1065680896 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034198595787362086, + "loss": 2.6685, + "theoretical_loss": 3.6272721806402983, + "tokens_seen": 1065746432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419759277833501, + "loss": 2.6327, + "theoretical_loss": 3.6272512033881945, + "tokens_seen": 1065811968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419658976930792, + "loss": 2.6649, + "theoretical_loss": 3.6272302277870665, + "tokens_seen": 1065877504 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034195586760280846, + "loss": 2.6261, + "theoretical_loss": 3.627209253836684, + "tokens_seen": 1065943040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419458375125376, + "loss": 2.7778, + "theoretical_loss": 3.627188281536814, + "tokens_seen": 1066008576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419358074222668, + "loss": 2.6386, + "theoretical_loss": 3.627167310887227, + "tokens_seen": 1066074112 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341925777331996, + "loss": 2.6872, + "theoretical_loss": 3.6271463418876904, + "tokens_seen": 1066139648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419157472417252, + "loss": 2.7311, + "theoretical_loss": 3.6271253745379726, + "tokens_seen": 1066205184 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034190571715145436, + "loss": 2.6386, + "theoretical_loss": 3.6271044088378437, + "tokens_seen": 1066270720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034189568706118355, + "loss": 2.5569, + "theoretical_loss": 3.627083444787072, + "tokens_seen": 1066336256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418856569709127, + "loss": 2.5788, + "theoretical_loss": 3.6270624823854263, + "tokens_seen": 1066401792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034187562688064196, + "loss": 2.4098, + "theoretical_loss": 3.6270415216326755, + "tokens_seen": 1066467328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418655967903711, + "loss": 2.4781, + "theoretical_loss": 3.6270205625285885, + "tokens_seen": 1066532864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 595571, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6795873641967773, + "objective/train/theoretical_loss": 3.6269996050729345, + "objective/train/tokens_used": 1087058400, + "theoretical_loss": 3.6269996050729345, + "tokens_seen": 1066598400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418555667001003, + "loss": 2.613, + "theoretical_loss": 3.6269996050729345, + "tokens_seen": 1066598400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034184553660982945, + "loss": 2.5394, + "theoretical_loss": 3.6269786492654825, + "tokens_seen": 1066663936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418355065195587, + "loss": 2.3815, + "theoretical_loss": 3.626957695106002, + "tokens_seen": 1066729472 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034182547642928787, + "loss": 2.4471, + "theoretical_loss": 3.6269367425942614, + "tokens_seen": 1066795008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034181544633901705, + "loss": 2.6606, + "theoretical_loss": 3.626915791730031, + "tokens_seen": 1066860544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034180541624874623, + "loss": 2.637, + "theoretical_loss": 3.626894842513079, + "tokens_seen": 1066926080 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034179538615847547, + "loss": 2.8521, + "theoretical_loss": 3.6268738949431754, + "tokens_seen": 1066991616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417853560682046, + "loss": 2.5181, + "theoretical_loss": 3.6268529490200887, + "tokens_seen": 1067057152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034177532597793383, + "loss": 2.6079, + "theoretical_loss": 3.626832004743589, + "tokens_seen": 1067122688 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034176529588766295, + "loss": 2.6194, + "theoretical_loss": 3.626811062113446, + "tokens_seen": 1067188224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417552657973922, + "loss": 2.6109, + "theoretical_loss": 3.626790121129429, + "tokens_seen": 1067253760 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034174523570712137, + "loss": 2.6758, + "theoretical_loss": 3.626769181791307, + "tokens_seen": 1067319296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034173520561685055, + "loss": 2.6405, + "theoretical_loss": 3.62674824409885, + "tokens_seen": 1067384832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034172517552657973, + "loss": 2.7384, + "theoretical_loss": 3.626727308051828, + "tokens_seen": 1067450368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417151454363089, + "loss": 2.7122, + "theoretical_loss": 3.6267063736500096, + "tokens_seen": 1067515904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417051153460381, + "loss": 2.741, + "theoretical_loss": 3.6266854408931657, + "tokens_seen": 1067581440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034169508525576733, + "loss": 2.5662, + "theoretical_loss": 3.6266645097810652, + "tokens_seen": 1067646976 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034168505516549646, + "loss": 2.4996, + "theoretical_loss": 3.6266435803134787, + "tokens_seen": 1067712512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416750250752257, + "loss": 2.5729, + "theoretical_loss": 3.6266226524901755, + "tokens_seen": 1067778048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416649949849548, + "loss": 2.738, + "theoretical_loss": 3.6266017263109256, + "tokens_seen": 1067843584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034165496489468406, + "loss": 2.7026, + "theoretical_loss": 3.626580801775499, + "tokens_seen": 1067909120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034164493480441324, + "loss": 2.3374, + "theoretical_loss": 3.6265598788836657, + "tokens_seen": 1067974656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416349047141424, + "loss": 2.6919, + "theoretical_loss": 3.626538957635196, + "tokens_seen": 1068040192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416248746238716, + "loss": 2.8102, + "theoretical_loss": 3.62651803802986, + "tokens_seen": 1068105728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034161484453360083, + "loss": 2.5649, + "theoretical_loss": 3.626497120067427, + "tokens_seen": 1068171264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 595858, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9211819171905518, + "objective/train/theoretical_loss": 3.6264762037476683, + "objective/train/tokens_used": 1088696800, + "theoretical_loss": 3.6264762037476683, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034160481444332996, + "loss": 2.5103, + "theoretical_loss": 3.6264762037476683, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415947843530592, + "loss": 2.6812, + "theoretical_loss": 3.6264552890703534, + "tokens_seen": 1068302336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415847542627884, + "loss": 2.6637, + "theoretical_loss": 3.6264343760352533, + "tokens_seen": 1068367872 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034157472417251756, + "loss": 2.7339, + "theoretical_loss": 3.6264134646421375, + "tokens_seen": 1068433408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415646940822468, + "loss": 2.6223, + "theoretical_loss": 3.6263925548907765, + "tokens_seen": 1068498944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415546639919759, + "loss": 2.6919, + "theoretical_loss": 3.6263716467809415, + "tokens_seen": 1068564480 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034154463390170516, + "loss": 2.7286, + "theoretical_loss": 3.626350740312402, + "tokens_seen": 1068630016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415346038114343, + "loss": 2.5326, + "theoretical_loss": 3.6263298354849294, + "tokens_seen": 1068695552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415245737211635, + "loss": 2.5669, + "theoretical_loss": 3.6263089322982935, + "tokens_seen": 1068761088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415145436308927, + "loss": 2.656, + "theoretical_loss": 3.6262880307522654, + "tokens_seen": 1068826624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415045135406219, + "loss": 2.5146, + "theoretical_loss": 3.626267130846615, + "tokens_seen": 1068892160 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034149448345035106, + "loss": 2.5452, + "theoretical_loss": 3.626246232581114, + "tokens_seen": 1068957696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414844533600803, + "loss": 2.5444, + "theoretical_loss": 3.6262253359555325, + "tokens_seen": 1069023232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414744232698094, + "loss": 2.507, + "theoretical_loss": 3.6262044409696417, + "tokens_seen": 1069088768 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034146439317953866, + "loss": 2.5204, + "theoretical_loss": 3.6261835476232123, + "tokens_seen": 1069154304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414543630892678, + "loss": 2.6365, + "theoretical_loss": 3.6261626559160147, + "tokens_seen": 1069219840 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341444332998997, + "loss": 2.5105, + "theoretical_loss": 3.6261417658478208, + "tokens_seen": 1069285376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414343029087262, + "loss": 2.7935, + "theoretical_loss": 3.6261208774184004, + "tokens_seen": 1069350912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414242728184554, + "loss": 2.575, + "theoretical_loss": 3.6260999906275257, + "tokens_seen": 1069416448 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034141424272818456, + "loss": 2.5472, + "theoretical_loss": 3.6260791054749673, + "tokens_seen": 1069481984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034140421263791375, + "loss": 2.5069, + "theoretical_loss": 3.6260582219604958, + "tokens_seen": 1069547520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034139418254764293, + "loss": 2.7579, + "theoretical_loss": 3.626037340083883, + "tokens_seen": 1069613056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034138415245737216, + "loss": 2.6508, + "theoretical_loss": 3.6260164598448994, + "tokens_seen": 1069678592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413741223671013, + "loss": 2.7109, + "theoretical_loss": 3.625995581243317, + "tokens_seen": 1069744128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413640922768305, + "loss": 2.6212, + "theoretical_loss": 3.6259747042789074, + "tokens_seen": 1069809664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 597140, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7327358722686768, + "objective/train/theoretical_loss": 3.625953828951441, + "objective/train/tokens_used": 1090335200, + "theoretical_loss": 3.625953828951441, + "tokens_seen": 1069875200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034135406218655965, + "loss": 2.6413, + "theoretical_loss": 3.625953828951441, + "tokens_seen": 1069875200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413440320962889, + "loss": 2.6578, + "theoretical_loss": 3.625932955260689, + "tokens_seen": 1069940736 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034133400200601807, + "loss": 2.6887, + "theoretical_loss": 3.625912083206424, + "tokens_seen": 1070006272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034132397191574725, + "loss": 2.3918, + "theoretical_loss": 3.625891212788417, + "tokens_seen": 1070071808 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034131394182547643, + "loss": 2.8379, + "theoretical_loss": 3.625870344006439, + "tokens_seen": 1070137344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034130391173520567, + "loss": 2.8266, + "theoretical_loss": 3.625849476860262, + "tokens_seen": 1070202880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412938816449348, + "loss": 2.6852, + "theoretical_loss": 3.6258286113496574, + "tokens_seen": 1070268416 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034128385155466403, + "loss": 2.5112, + "theoretical_loss": 3.6258077474743975, + "tokens_seen": 1070333952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034127382146439315, + "loss": 2.6649, + "theoretical_loss": 3.625786885234253, + "tokens_seen": 1070399488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412637913741224, + "loss": 2.5514, + "theoretical_loss": 3.625766024628997, + "tokens_seen": 1070465024 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034125376128385157, + "loss": 2.7262, + "theoretical_loss": 3.6257451656584, + "tokens_seen": 1070530560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034124373119358075, + "loss": 2.6269, + "theoretical_loss": 3.6257243083222344, + "tokens_seen": 1070596096 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034123370110330993, + "loss": 2.5139, + "theoretical_loss": 3.6257034526202725, + "tokens_seen": 1070661632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412236710130391, + "loss": 2.7592, + "theoretical_loss": 3.6256825985522854, + "tokens_seen": 1070727168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412136409227683, + "loss": 2.6409, + "theoretical_loss": 3.6256617461180456, + "tokens_seen": 1070792704 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034120361083249753, + "loss": 2.6633, + "theoretical_loss": 3.625640895317325, + "tokens_seen": 1070858240 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034119358074222666, + "loss": 2.5714, + "theoretical_loss": 3.625620046149895, + "tokens_seen": 1070923776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411835506519559, + "loss": 2.6975, + "theoretical_loss": 3.6255991986155296, + "tokens_seen": 1070989312 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341173520561685, + "loss": 2.8864, + "theoretical_loss": 3.625578352713999, + "tokens_seen": 1071054848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034116349047141426, + "loss": 2.8835, + "theoretical_loss": 3.625557508445076, + "tokens_seen": 1071120384 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034115346038114344, + "loss": 2.434, + "theoretical_loss": 3.625536665808534, + "tokens_seen": 1071185920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411434302908726, + "loss": 2.7717, + "theoretical_loss": 3.6255158248041433, + "tokens_seen": 1071251456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411334002006018, + "loss": 2.6301, + "theoretical_loss": 3.6254949854316783, + "tokens_seen": 1071316992 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034112337011033103, + "loss": 2.6367, + "theoretical_loss": 3.62547414769091, + "tokens_seen": 1071382528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034111334002006016, + "loss": 2.5804, + "theoretical_loss": 3.6254533115816105, + "tokens_seen": 1071448064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 598183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3538804054260254, + "objective/train/theoretical_loss": 3.625432477103554, + "objective/train/tokens_used": 1091973600, + "theoretical_loss": 3.625432477103554, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411033099297894, + "loss": 2.3987, + "theoretical_loss": 3.625432477103554, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410932798395185, + "loss": 2.6763, + "theoretical_loss": 3.6254116442565114, + "tokens_seen": 1071579136 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034108324974924776, + "loss": 2.8075, + "theoretical_loss": 3.625390813040257, + "tokens_seen": 1071644672 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034107321965897694, + "loss": 2.6814, + "theoretical_loss": 3.625369983454561, + "tokens_seen": 1071710208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410631895687061, + "loss": 2.6329, + "theoretical_loss": 3.6253491554991983, + "tokens_seen": 1071775744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410531594784353, + "loss": 2.3829, + "theoretical_loss": 3.62532832917394, + "tokens_seen": 1071841280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410431293881645, + "loss": 2.6071, + "theoretical_loss": 3.6253075044785605, + "tokens_seen": 1071906816 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034103309929789366, + "loss": 2.7892, + "theoretical_loss": 3.625286681412831, + "tokens_seen": 1071972352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410230692076229, + "loss": 2.7847, + "theoretical_loss": 3.6252658599765257, + "tokens_seen": 1072037888 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341013039117352, + "loss": 2.6912, + "theoretical_loss": 3.6252450401694167, + "tokens_seen": 1072103424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034100300902708126, + "loss": 2.5225, + "theoretical_loss": 3.6252242219912767, + "tokens_seen": 1072168960 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034099297893681044, + "loss": 2.6905, + "theoretical_loss": 3.6252034054418796, + "tokens_seen": 1072234496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409829488465396, + "loss": 2.7699, + "theoretical_loss": 3.6251825905209976, + "tokens_seen": 1072300032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409729187562688, + "loss": 2.4818, + "theoretical_loss": 3.625161777228404, + "tokens_seen": 1072365568 + }, + { + "epoch": 0.32, + "learning_rate": 0.000340962888665998, + "loss": 2.8261, + "theoretical_loss": 3.6251409655638724, + "tokens_seen": 1072431104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034095285857572717, + "loss": 2.7078, + "theoretical_loss": 3.6251201555271755, + "tokens_seen": 1072496640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003409428284854564, + "loss": 2.5445, + "theoretical_loss": 3.625099347118087, + "tokens_seen": 1072562176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034093279839518553, + "loss": 2.6212, + "theoretical_loss": 3.625078540336379, + "tokens_seen": 1072627712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034092276830491477, + "loss": 2.5129, + "theoretical_loss": 3.625057735181826, + "tokens_seen": 1072693248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003409127382146439, + "loss": 2.6918, + "theoretical_loss": 3.6250369316542006, + "tokens_seen": 1072758784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034090270812437313, + "loss": 2.4667, + "theoretical_loss": 3.625016129753277, + "tokens_seen": 1072824320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408926780341023, + "loss": 2.4542, + "theoretical_loss": 3.6249953294788284, + "tokens_seen": 1072889856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408826479438315, + "loss": 2.6622, + "theoretical_loss": 3.6249745308306274, + "tokens_seen": 1072955392 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034087261785356067, + "loss": 2.7896, + "theoretical_loss": 3.6249537338084488, + "tokens_seen": 1073020928 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034086258776328985, + "loss": 2.7108, + "theoretical_loss": 3.624932938412065, + "tokens_seen": 1073086464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 598781, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9608399868011475, + "objective/train/theoretical_loss": 3.6249121446412507, + "objective/train/tokens_used": 1093612000, + "theoretical_loss": 3.6249121446412507, + "tokens_seen": 1073152000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034085255767301903, + "loss": 2.7379, + "theoretical_loss": 3.6249121446412507, + "tokens_seen": 1073152000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034084252758274827, + "loss": 2.5467, + "theoretical_loss": 3.624891352495779, + "tokens_seen": 1073217536 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034083249749247745, + "loss": 2.3039, + "theoretical_loss": 3.624870561975423, + "tokens_seen": 1073283072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034082246740220663, + "loss": 2.6401, + "theoretical_loss": 3.6248497730799585, + "tokens_seen": 1073348608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034081243731193587, + "loss": 2.4763, + "theoretical_loss": 3.624828985809157, + "tokens_seen": 1073414144 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340802407221665, + "loss": 2.5679, + "theoretical_loss": 3.6248082001627937, + "tokens_seen": 1073479680 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034079237713139423, + "loss": 2.7953, + "theoretical_loss": 3.6247874161406415, + "tokens_seen": 1073545216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034078234704112335, + "loss": 2.6214, + "theoretical_loss": 3.6247666337424755, + "tokens_seen": 1073610752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407723169508526, + "loss": 2.6529, + "theoretical_loss": 3.6247458529680694, + "tokens_seen": 1073676288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034076228686058177, + "loss": 2.7148, + "theoretical_loss": 3.6247250738171966, + "tokens_seen": 1073741824 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034075225677031095, + "loss": 2.9034, + "theoretical_loss": 3.6247042962896314, + "tokens_seen": 1073807360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034074222668004013, + "loss": 2.6231, + "theoretical_loss": 3.6246835203851484, + "tokens_seen": 1073872896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407321965897693, + "loss": 2.6436, + "theoretical_loss": 3.624662746103521, + "tokens_seen": 1073938432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407221664994985, + "loss": 2.5244, + "theoretical_loss": 3.6246419734445245, + "tokens_seen": 1074003968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034071213640922773, + "loss": 2.6201, + "theoretical_loss": 3.624621202407932, + "tokens_seen": 1074069504 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034070210631895686, + "loss": 2.5998, + "theoretical_loss": 3.6246004329935184, + "tokens_seen": 1074135040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406920762286861, + "loss": 2.4807, + "theoretical_loss": 3.6245796652010585, + "tokens_seen": 1074200576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406820461384152, + "loss": 2.7288, + "theoretical_loss": 3.6245588990303252, + "tokens_seen": 1074266112 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034067201604814446, + "loss": 2.4171, + "theoretical_loss": 3.624538134481095, + "tokens_seen": 1074331648 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034066198595787364, + "loss": 2.6719, + "theoretical_loss": 3.6245173715531402, + "tokens_seen": 1074397184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406519558676028, + "loss": 2.6694, + "theoretical_loss": 3.6244966102462373, + "tokens_seen": 1074462720 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340641925777332, + "loss": 2.8719, + "theoretical_loss": 3.6244758505601595, + "tokens_seen": 1074528256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034063189568706123, + "loss": 2.687, + "theoretical_loss": 3.624455092494682, + "tokens_seen": 1074593792 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034062186559679036, + "loss": 2.7869, + "theoretical_loss": 3.624434336049579, + "tokens_seen": 1074659328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406118355065196, + "loss": 2.5128, + "theoretical_loss": 3.6244135812246263, + "tokens_seen": 1074724864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 599993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5279252529144287, + "objective/train/theoretical_loss": 3.6243928280195976, + "objective/train/tokens_used": 1095250400, + "theoretical_loss": 3.6243928280195976, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406018054162487, + "loss": 2.6169, + "theoretical_loss": 3.6243928280195976, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034059177532597796, + "loss": 2.6747, + "theoretical_loss": 3.624372076434268, + "tokens_seen": 1074855936 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034058174523570714, + "loss": 2.7016, + "theoretical_loss": 3.6243513264684126, + "tokens_seen": 1074921472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405717151454363, + "loss": 2.7665, + "theoretical_loss": 3.624330578121806, + "tokens_seen": 1074987008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405616850551655, + "loss": 2.859, + "theoretical_loss": 3.624309831394223, + "tokens_seen": 1075052544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405516549648947, + "loss": 2.8091, + "theoretical_loss": 3.624289086285439, + "tokens_seen": 1075118080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034054162487462386, + "loss": 2.787, + "theoretical_loss": 3.6242683427952285, + "tokens_seen": 1075183616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405315947843531, + "loss": 2.6744, + "theoretical_loss": 3.6242476009233675, + "tokens_seen": 1075249152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405215646940822, + "loss": 2.6618, + "theoretical_loss": 3.62422686066963, + "tokens_seen": 1075314688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034051153460381146, + "loss": 2.4714, + "theoretical_loss": 3.6242061220337916, + "tokens_seen": 1075380224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034050150451354064, + "loss": 2.6045, + "theoretical_loss": 3.624185385015628, + "tokens_seen": 1075445760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404914744232698, + "loss": 2.5883, + "theoretical_loss": 3.6241646496149134, + "tokens_seen": 1075511296 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340481444332999, + "loss": 2.7271, + "theoretical_loss": 3.624143915831424, + "tokens_seen": 1075576832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404714142427282, + "loss": 2.7838, + "theoretical_loss": 3.624123183664935, + "tokens_seen": 1075642368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034046138415245737, + "loss": 2.7861, + "theoretical_loss": 3.6241024531152215, + "tokens_seen": 1075707904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404513540621866, + "loss": 2.7807, + "theoretical_loss": 3.6240817241820587, + "tokens_seen": 1075773440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034044132397191573, + "loss": 2.7066, + "theoretical_loss": 3.624060996865223, + "tokens_seen": 1075838976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034043129388164497, + "loss": 2.6692, + "theoretical_loss": 3.624040271164489, + "tokens_seen": 1075904512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404212637913741, + "loss": 2.6054, + "theoretical_loss": 3.624019547079633, + "tokens_seen": 1075970048 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034041123370110333, + "loss": 2.7437, + "theoretical_loss": 3.6239988246104295, + "tokens_seen": 1076035584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404012036108325, + "loss": 2.651, + "theoretical_loss": 3.6239781037566554, + "tokens_seen": 1076101120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403911735205617, + "loss": 2.6356, + "theoretical_loss": 3.6239573845180857, + "tokens_seen": 1076166656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034038114343029087, + "loss": 2.4081, + "theoretical_loss": 3.6239366668944966, + "tokens_seen": 1076232192 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034037111334002005, + "loss": 2.6507, + "theoretical_loss": 3.6239159508856638, + "tokens_seen": 1076297728 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034036108324974923, + "loss": 2.6173, + "theoretical_loss": 3.6238952364913626, + "tokens_seen": 1076363264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 601033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.560241460800171, + "objective/train/theoretical_loss": 3.623874523711369, + "objective/train/tokens_used": 1096888800, + "theoretical_loss": 3.623874523711369, + "tokens_seen": 1076428800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034035105315947847, + "loss": 2.4641, + "theoretical_loss": 3.623874523711369, + "tokens_seen": 1076428800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403410230692076, + "loss": 2.9075, + "theoretical_loss": 3.6238538125454594, + "tokens_seen": 1076494336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034033099297893683, + "loss": 2.7106, + "theoretical_loss": 3.623833102993409, + "tokens_seen": 1076559872 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340320962888666, + "loss": 2.5626, + "theoretical_loss": 3.623812395054995, + "tokens_seen": 1076625408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403109327983952, + "loss": 2.667, + "theoretical_loss": 3.6237916887299924, + "tokens_seen": 1076690944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403009027081244, + "loss": 2.4823, + "theoretical_loss": 3.623770984018178, + "tokens_seen": 1076756480 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034029087261785356, + "loss": 2.514, + "theoretical_loss": 3.623750280919327, + "tokens_seen": 1076822016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034028084252758274, + "loss": 2.8244, + "theoretical_loss": 3.623729579433217, + "tokens_seen": 1076887552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034027081243731197, + "loss": 2.6843, + "theoretical_loss": 3.623708879559623, + "tokens_seen": 1076953088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402607823470411, + "loss": 2.663, + "theoretical_loss": 3.6236881812983217, + "tokens_seen": 1077018624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034025075225677033, + "loss": 2.7929, + "theoretical_loss": 3.62366748464909, + "tokens_seen": 1077084160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034024072216649946, + "loss": 2.5873, + "theoretical_loss": 3.623646789611703, + "tokens_seen": 1077149696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402306920762287, + "loss": 2.5329, + "theoretical_loss": 3.623626096185938, + "tokens_seen": 1077215232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402206619859579, + "loss": 2.8096, + "theoretical_loss": 3.6236054043715713, + "tokens_seen": 1077280768 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034021063189568706, + "loss": 2.4211, + "theoretical_loss": 3.6235847141683797, + "tokens_seen": 1077346304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034020060180541624, + "loss": 2.6624, + "theoretical_loss": 3.623564025576139, + "tokens_seen": 1077411840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401905717151454, + "loss": 2.9951, + "theoretical_loss": 3.6235433385946267, + "tokens_seen": 1077477376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401805416248746, + "loss": 2.6661, + "theoretical_loss": 3.6235226532236187, + "tokens_seen": 1077542912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034017051153460384, + "loss": 2.5833, + "theoretical_loss": 3.623501969462892, + "tokens_seen": 1077608448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034016048144433296, + "loss": 2.7677, + "theoretical_loss": 3.623481287312223, + "tokens_seen": 1077673984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401504513540622, + "loss": 2.5585, + "theoretical_loss": 3.6234606067713893, + "tokens_seen": 1077739520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401404212637914, + "loss": 2.8325, + "theoretical_loss": 3.6234399278401668, + "tokens_seen": 1077805056 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034013039117352056, + "loss": 2.5578, + "theoretical_loss": 3.6234192505183325, + "tokens_seen": 1077870592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034012036108324974, + "loss": 2.8148, + "theoretical_loss": 3.623398574805664, + "tokens_seen": 1077936128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401103309929789, + "loss": 2.8187, + "theoretical_loss": 3.6233779007019375, + "tokens_seen": 1078001664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 601484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.654538869857788, + "objective/train/theoretical_loss": 3.62335722820693, + "objective/train/tokens_used": 1098527200, + "theoretical_loss": 3.62335722820693, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401003009027081, + "loss": 2.5519, + "theoretical_loss": 3.62335722820693, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034009027081243734, + "loss": 2.8683, + "theoretical_loss": 3.6233365573204193, + "tokens_seen": 1078132736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400802407221665, + "loss": 2.7965, + "theoretical_loss": 3.623315888042182, + "tokens_seen": 1078198272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400702106318957, + "loss": 2.8259, + "theoretical_loss": 3.623295220371995, + "tokens_seen": 1078263808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400601805416249, + "loss": 2.5405, + "theoretical_loss": 3.623274554309636, + "tokens_seen": 1078329344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034005015045135406, + "loss": 2.4842, + "theoretical_loss": 3.6232538898548814, + "tokens_seen": 1078394880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400401203610833, + "loss": 2.6957, + "theoretical_loss": 3.623233227007509, + "tokens_seen": 1078460416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400300902708124, + "loss": 2.8692, + "theoretical_loss": 3.6232125657672962, + "tokens_seen": 1078525952 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034002006018054166, + "loss": 2.7149, + "theoretical_loss": 3.6231919061340205, + "tokens_seen": 1078591488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034001003009027084, + "loss": 2.6374, + "theoretical_loss": 3.6231712481074583, + "tokens_seen": 1078657024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034, + "loss": 2.5463, + "theoretical_loss": 3.6231505916873887, + "tokens_seen": 1078722560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399899699097292, + "loss": 2.6222, + "theoretical_loss": 3.623129936873587, + "tokens_seen": 1078788096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399799398194584, + "loss": 2.9549, + "theoretical_loss": 3.623109283665833, + "tokens_seen": 1078853632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033996990972918757, + "loss": 2.5965, + "theoretical_loss": 3.6230886320639026, + "tokens_seen": 1078919168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399598796389168, + "loss": 2.7922, + "theoretical_loss": 3.6230679820675746, + "tokens_seen": 1078984704 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033994984954864593, + "loss": 2.7641, + "theoretical_loss": 3.6230473336766256, + "tokens_seen": 1079050240 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033993981945837517, + "loss": 2.5767, + "theoretical_loss": 3.6230266868908343, + "tokens_seen": 1079115776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399297893681043, + "loss": 2.7457, + "theoretical_loss": 3.6230060417099774, + "tokens_seen": 1079181312 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033991975927783353, + "loss": 2.7767, + "theoretical_loss": 3.6229853981338334, + "tokens_seen": 1079246848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399097291875627, + "loss": 2.5381, + "theoretical_loss": 3.62296475616218, + "tokens_seen": 1079312384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398996990972919, + "loss": 2.7448, + "theoretical_loss": 3.622944115794795, + "tokens_seen": 1079377920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033988966900702107, + "loss": 2.8126, + "theoretical_loss": 3.6229234770314562, + "tokens_seen": 1079443456 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033987963891675025, + "loss": 2.7071, + "theoretical_loss": 3.622902839871942, + "tokens_seen": 1079508992 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033986960882647943, + "loss": 2.8773, + "theoretical_loss": 3.6228822043160305, + "tokens_seen": 1079574528 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033985957873620867, + "loss": 2.6932, + "theoretical_loss": 3.622861570363499, + "tokens_seen": 1079640064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 602559, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.677448034286499, + "objective/train/theoretical_loss": 3.622840938014126, + "objective/train/tokens_used": 1100165600, + "theoretical_loss": 3.622840938014126, + "tokens_seen": 1079705600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398495486459378, + "loss": 2.611, + "theoretical_loss": 3.622840938014126, + "tokens_seen": 1079705600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033983951855566703, + "loss": 2.4328, + "theoretical_loss": 3.6228203072676894, + "tokens_seen": 1079771136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398294884653962, + "loss": 2.6645, + "theoretical_loss": 3.622799678123968, + "tokens_seen": 1079836672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398194583751254, + "loss": 2.8604, + "theoretical_loss": 3.6227790505827393, + "tokens_seen": 1079902208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398094282848546, + "loss": 2.6952, + "theoretical_loss": 3.6227584246437825, + "tokens_seen": 1079967744 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033979939819458376, + "loss": 2.6203, + "theoretical_loss": 3.622737800306875, + "tokens_seen": 1080033280 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033978936810431294, + "loss": 2.6824, + "theoretical_loss": 3.6227171775717952, + "tokens_seen": 1080098816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033977933801404217, + "loss": 2.4892, + "theoretical_loss": 3.6226965564383224, + "tokens_seen": 1080164352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397693079237713, + "loss": 2.8857, + "theoretical_loss": 3.6226759369062345, + "tokens_seen": 1080229888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033975927783350053, + "loss": 2.5533, + "theoretical_loss": 3.62265531897531, + "tokens_seen": 1080295424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033974924774322966, + "loss": 2.7272, + "theoretical_loss": 3.6226347026453274, + "tokens_seen": 1080360960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397392176529589, + "loss": 2.8356, + "theoretical_loss": 3.622614087916065, + "tokens_seen": 1080426496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397291875626881, + "loss": 2.6642, + "theoretical_loss": 3.622593474787302, + "tokens_seen": 1080492032 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033971915747241726, + "loss": 2.7828, + "theoretical_loss": 3.622572863258817, + "tokens_seen": 1080557568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033970912738214644, + "loss": 2.7391, + "theoretical_loss": 3.6225522533303884, + "tokens_seen": 1080623104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396990972918756, + "loss": 2.7628, + "theoretical_loss": 3.622531645001795, + "tokens_seen": 1080688640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396890672016048, + "loss": 2.6882, + "theoretical_loss": 3.6225110382728163, + "tokens_seen": 1080754176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033967903711133404, + "loss": 2.7996, + "theoretical_loss": 3.6224904331432297, + "tokens_seen": 1080819712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033966900702106316, + "loss": 2.74, + "theoretical_loss": 3.6224698296128155, + "tokens_seen": 1080885248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396589769307924, + "loss": 2.9426, + "theoretical_loss": 3.6224492276813516, + "tokens_seen": 1080950784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396489468405216, + "loss": 2.6184, + "theoretical_loss": 3.622428627348618, + "tokens_seen": 1081016320 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033963891675025076, + "loss": 2.6755, + "theoretical_loss": 3.6224080286143936, + "tokens_seen": 1081081856 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033962888665997994, + "loss": 2.8268, + "theoretical_loss": 3.6223874314784563, + "tokens_seen": 1081147392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396188565697091, + "loss": 2.6844, + "theoretical_loss": 3.622366835940586, + "tokens_seen": 1081212928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396088264794383, + "loss": 2.6764, + "theoretical_loss": 3.622346242000562, + "tokens_seen": 1081278464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 603206, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.54484224319458, + "objective/train/theoretical_loss": 3.6223256496581637, + "objective/train/tokens_used": 1101804000, + "theoretical_loss": 3.6223256496581637, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033959879638916754, + "loss": 2.7759, + "theoretical_loss": 3.6223256496581637, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033958876629889667, + "loss": 2.7449, + "theoretical_loss": 3.62230505891317, + "tokens_seen": 1081409536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395787362086259, + "loss": 2.8069, + "theoretical_loss": 3.6222844697653596, + "tokens_seen": 1081475072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033956870611835503, + "loss": 2.567, + "theoretical_loss": 3.6222638822145132, + "tokens_seen": 1081540608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033955867602808426, + "loss": 2.7481, + "theoretical_loss": 3.6222432962604088, + "tokens_seen": 1081606144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033954864593781345, + "loss": 2.5326, + "theoretical_loss": 3.6222227119028263, + "tokens_seen": 1081671680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395386158475426, + "loss": 2.5933, + "theoretical_loss": 3.622202129141546, + "tokens_seen": 1081737216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395285857572718, + "loss": 2.7438, + "theoretical_loss": 3.6221815479763464, + "tokens_seen": 1081802752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033951855566700104, + "loss": 2.4687, + "theoretical_loss": 3.622160968407007, + "tokens_seen": 1081868288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033950852557673017, + "loss": 2.7814, + "theoretical_loss": 3.622140390433308, + "tokens_seen": 1081933824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394984954864594, + "loss": 2.5706, + "theoretical_loss": 3.622119814055029, + "tokens_seen": 1081999360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033948846539618853, + "loss": 2.7052, + "theoretical_loss": 3.6220992392719493, + "tokens_seen": 1082064896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033947843530591777, + "loss": 2.5586, + "theoretical_loss": 3.6220786660838487, + "tokens_seen": 1082130432 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033946840521564695, + "loss": 2.676, + "theoretical_loss": 3.622058094490507, + "tokens_seen": 1082195968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033945837512537613, + "loss": 2.8831, + "theoretical_loss": 3.622037524491704, + "tokens_seen": 1082261504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394483450351053, + "loss": 2.6192, + "theoretical_loss": 3.6220169560872204, + "tokens_seen": 1082327040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394383149448345, + "loss": 2.6554, + "theoretical_loss": 3.6219963892768345, + "tokens_seen": 1082392576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394282848545637, + "loss": 2.5606, + "theoretical_loss": 3.6219758240603275, + "tokens_seen": 1082458112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394182547642929, + "loss": 2.7442, + "theoretical_loss": 3.6219552604374785, + "tokens_seen": 1082523648 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033940822467402204, + "loss": 2.5597, + "theoretical_loss": 3.6219346984080687, + "tokens_seen": 1082589184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033939819458375127, + "loss": 2.7137, + "theoretical_loss": 3.621914137971877, + "tokens_seen": 1082654720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393881644934804, + "loss": 2.8618, + "theoretical_loss": 3.6218935791286846, + "tokens_seen": 1082720256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033937813440320963, + "loss": 2.5351, + "theoretical_loss": 3.6218730218782706, + "tokens_seen": 1082785792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393681043129388, + "loss": 2.7446, + "theoretical_loss": 3.6218524662204157, + "tokens_seen": 1082851328 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339358074222668, + "loss": 2.6969, + "theoretical_loss": 3.6218319121549003, + "tokens_seen": 1082916864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 603451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.421107530593872, + "objective/train/theoretical_loss": 3.6218113596815042, + "objective/train/tokens_used": 1103442400, + "theoretical_loss": 3.6218113596815042, + "tokens_seen": 1082982400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393480441323972, + "loss": 2.5057, + "theoretical_loss": 3.6218113596815042, + "tokens_seen": 1082982400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393380140421264, + "loss": 2.6102, + "theoretical_loss": 3.6217908088000086, + "tokens_seen": 1083047936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393279839518556, + "loss": 2.8188, + "theoretical_loss": 3.6217702595101935, + "tokens_seen": 1083113472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393179538615848, + "loss": 2.6628, + "theoretical_loss": 3.6217497118118382, + "tokens_seen": 1083179008 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033930792377131396, + "loss": 2.6231, + "theoretical_loss": 3.621729165704725, + "tokens_seen": 1083244544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033929789368104314, + "loss": 2.3983, + "theoretical_loss": 3.6217086211886333, + "tokens_seen": 1083310080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033928786359077237, + "loss": 2.6967, + "theoretical_loss": 3.621688078263344, + "tokens_seen": 1083375616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392778335005015, + "loss": 2.74, + "theoretical_loss": 3.6216675369286375, + "tokens_seen": 1083441152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033926780341023073, + "loss": 2.7629, + "theoretical_loss": 3.621646997184295, + "tokens_seen": 1083506688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033925777331995986, + "loss": 2.6403, + "theoretical_loss": 3.6216264590300966, + "tokens_seen": 1083572224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392477432296891, + "loss": 2.8355, + "theoretical_loss": 3.6216059224658226, + "tokens_seen": 1083637760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392377131394183, + "loss": 2.7227, + "theoretical_loss": 3.6215853874912547, + "tokens_seen": 1083703296 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033922768304914746, + "loss": 2.7424, + "theoretical_loss": 3.6215648541061736, + "tokens_seen": 1083768832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033921765295887664, + "loss": 2.6606, + "theoretical_loss": 3.62154432231036, + "tokens_seen": 1083834368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392076228686058, + "loss": 2.7373, + "theoretical_loss": 3.6215237921035945, + "tokens_seen": 1083899904 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339197592778335, + "loss": 2.5256, + "theoretical_loss": 3.6215032634856583, + "tokens_seen": 1083965440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033918756268806424, + "loss": 2.7521, + "theoretical_loss": 3.6214827364563327, + "tokens_seen": 1084030976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033917753259779336, + "loss": 2.7552, + "theoretical_loss": 3.6214622110153982, + "tokens_seen": 1084096512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391675025075226, + "loss": 2.5513, + "theoretical_loss": 3.621441687162636, + "tokens_seen": 1084162048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391574724172518, + "loss": 2.7222, + "theoretical_loss": 3.6214211648978276, + "tokens_seen": 1084227584 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033914744232698096, + "loss": 2.6685, + "theoretical_loss": 3.6214006442207536, + "tokens_seen": 1084293120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033913741223671014, + "loss": 2.5153, + "theoretical_loss": 3.6213801251311954, + "tokens_seen": 1084358656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391273821464393, + "loss": 2.5689, + "theoretical_loss": 3.6213596076289347, + "tokens_seen": 1084424192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391173520561685, + "loss": 2.7692, + "theoretical_loss": 3.6213390917137525, + "tokens_seen": 1084489728 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033910732196589774, + "loss": 2.5669, + "theoretical_loss": 3.6213185773854297, + "tokens_seen": 1084555264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 603451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2022194862365723, + "objective/train/theoretical_loss": 3.6212980646437485, + "objective/train/tokens_used": 1105080800, + "theoretical_loss": 3.6212980646437485, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033909729187562687, + "loss": 2.8285, + "theoretical_loss": 3.6212980646437485, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390872617853561, + "loss": 2.5917, + "theoretical_loss": 3.62127755348849, + "tokens_seen": 1084686336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033907723169508523, + "loss": 2.5659, + "theoretical_loss": 3.621257043919435, + "tokens_seen": 1084751872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033906720160481446, + "loss": 2.8947, + "theoretical_loss": 3.621236535936366, + "tokens_seen": 1084817408 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033905717151454365, + "loss": 2.7032, + "theoretical_loss": 3.621216029539064, + "tokens_seen": 1084882944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033904714142427283, + "loss": 2.854, + "theoretical_loss": 3.6211955247273107, + "tokens_seen": 1084948480 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339037111334002, + "loss": 2.9125, + "theoretical_loss": 3.6211750215008878, + "tokens_seen": 1085014016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033902708124373124, + "loss": 2.6352, + "theoretical_loss": 3.6211545198595774, + "tokens_seen": 1085079552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033901705115346037, + "loss": 2.7928, + "theoretical_loss": 3.62113401980316, + "tokens_seen": 1085145088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390070210631896, + "loss": 2.8488, + "theoretical_loss": 3.6211135213314183, + "tokens_seen": 1085210624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033899699097291873, + "loss": 2.7162, + "theoretical_loss": 3.621093024444135, + "tokens_seen": 1085276160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033898696088264797, + "loss": 2.9749, + "theoretical_loss": 3.62107252914109, + "tokens_seen": 1085341696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033897693079237715, + "loss": 2.8548, + "theoretical_loss": 3.621052035422066, + "tokens_seen": 1085407232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033896690070210633, + "loss": 2.6953, + "theoretical_loss": 3.621031543286845, + "tokens_seen": 1085472768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389568706118355, + "loss": 2.8561, + "theoretical_loss": 3.62101105273521, + "tokens_seen": 1085538304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389468405215647, + "loss": 2.7889, + "theoretical_loss": 3.6209905637669415, + "tokens_seen": 1085603840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389368104312939, + "loss": 2.7944, + "theoretical_loss": 3.6209700763818224, + "tokens_seen": 1085669376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389267803410231, + "loss": 2.4984, + "theoretical_loss": 3.6209495905796345, + "tokens_seen": 1085734912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033891675025075224, + "loss": 2.8853, + "theoretical_loss": 3.62092910636016, + "tokens_seen": 1085800448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033890672016048147, + "loss": 2.8857, + "theoretical_loss": 3.6209086237231816, + "tokens_seen": 1085865984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388966900702106, + "loss": 2.8294, + "theoretical_loss": 3.6208881426684805, + "tokens_seen": 1085931520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033888665997993983, + "loss": 2.6671, + "theoretical_loss": 3.6208676631958396, + "tokens_seen": 1085997056 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338876629889669, + "loss": 2.7934, + "theoretical_loss": 3.6208471853050415, + "tokens_seen": 1086062592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388665997993982, + "loss": 2.6889, + "theoretical_loss": 3.6208267089958683, + "tokens_seen": 1086128128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388565697091274, + "loss": 2.6472, + "theoretical_loss": 3.6208062342681027, + "tokens_seen": 1086193664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 604042, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7982475757598877, + "objective/train/theoretical_loss": 3.6207857611215264, + "objective/train/tokens_used": 1106719200, + "theoretical_loss": 3.6207857611215264, + "tokens_seen": 1086259200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388465396188566, + "loss": 2.6263, + "theoretical_loss": 3.6207857611215264, + "tokens_seen": 1086259200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033883650952858574, + "loss": 2.8603, + "theoretical_loss": 3.6207652895559224, + "tokens_seen": 1086324736 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338826479438315, + "loss": 2.5424, + "theoretical_loss": 3.6207448195710734, + "tokens_seen": 1086390272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388164493480441, + "loss": 2.5165, + "theoretical_loss": 3.620724351166762, + "tokens_seen": 1086455808 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033880641925777334, + "loss": 2.7872, + "theoretical_loss": 3.6207038843427704, + "tokens_seen": 1086521344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387963891675025, + "loss": 2.4773, + "theoretical_loss": 3.6206834190988815, + "tokens_seen": 1086586880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387863590772317, + "loss": 2.7559, + "theoretical_loss": 3.620662955434878, + "tokens_seen": 1086652416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387763289869609, + "loss": 2.975, + "theoretical_loss": 3.6206424933505437, + "tokens_seen": 1086717952 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033876629889669006, + "loss": 2.9142, + "theoretical_loss": 3.6206220328456595, + "tokens_seen": 1086783488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033875626880641924, + "loss": 2.8429, + "theoretical_loss": 3.620601573920009, + "tokens_seen": 1086849024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387462387161485, + "loss": 2.6618, + "theoretical_loss": 3.620581116573376, + "tokens_seen": 1086914560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387362086258776, + "loss": 2.7312, + "theoretical_loss": 3.620560660805542, + "tokens_seen": 1086980096 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033872617853560684, + "loss": 2.5756, + "theoretical_loss": 3.6205402066162913, + "tokens_seen": 1087045632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033871614844533597, + "loss": 2.7202, + "theoretical_loss": 3.6205197540054064, + "tokens_seen": 1087111168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387061183550652, + "loss": 2.7871, + "theoretical_loss": 3.6204993029726698, + "tokens_seen": 1087176704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386960882647944, + "loss": 2.8353, + "theoretical_loss": 3.6204788535178656, + "tokens_seen": 1087242240 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033868605817452356, + "loss": 2.6144, + "theoretical_loss": 3.620458405640776, + "tokens_seen": 1087307776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033867602808425275, + "loss": 2.8865, + "theoretical_loss": 3.6204379593411846, + "tokens_seen": 1087373312 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338665997993982, + "loss": 2.8391, + "theoretical_loss": 3.6204175146188744, + "tokens_seen": 1087438848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386559679037111, + "loss": 2.7494, + "theoretical_loss": 3.6203970714736298, + "tokens_seen": 1087504384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033864593781344034, + "loss": 2.806, + "theoretical_loss": 3.6203766299052327, + "tokens_seen": 1087569920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033863590772316947, + "loss": 2.6098, + "theoretical_loss": 3.6203561899134673, + "tokens_seen": 1087635456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386258776328987, + "loss": 3.0417, + "theoretical_loss": 3.620335751498116, + "tokens_seen": 1087700992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386158475426279, + "loss": 2.9476, + "theoretical_loss": 3.6203153146589635, + "tokens_seen": 1087766528 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033860581745235707, + "loss": 2.6936, + "theoretical_loss": 3.620294879395793, + "tokens_seen": 1087832064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 604895, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.194870710372925, + "objective/train/theoretical_loss": 3.6202744457083877, + "objective/train/tokens_used": 1108357600, + "theoretical_loss": 3.6202744457083877, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033859578736208625, + "loss": 2.5966, + "theoretical_loss": 3.6202744457083877, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033858575727181543, + "loss": 2.8116, + "theoretical_loss": 3.620254013596531, + "tokens_seen": 1087963136 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033857572718154467, + "loss": 2.8302, + "theoretical_loss": 3.620233583060007, + "tokens_seen": 1088028672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033856569709127385, + "loss": 2.77, + "theoretical_loss": 3.6202131540985993, + "tokens_seen": 1088094208 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033855566700100303, + "loss": 2.8116, + "theoretical_loss": 3.6201927267120917, + "tokens_seen": 1088159744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385456369107322, + "loss": 2.9741, + "theoretical_loss": 3.6201723009002675, + "tokens_seen": 1088225280 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033853560682046144, + "loss": 2.8073, + "theoretical_loss": 3.62015187666291, + "tokens_seen": 1088290816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033852557673019057, + "loss": 2.5589, + "theoretical_loss": 3.6201314539998046, + "tokens_seen": 1088356352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385155466399198, + "loss": 2.812, + "theoretical_loss": 3.620111032910734, + "tokens_seen": 1088421888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033850551654964893, + "loss": 2.8191, + "theoretical_loss": 3.6200906133954827, + "tokens_seen": 1088487424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033849548645937817, + "loss": 2.9442, + "theoretical_loss": 3.620070195453834, + "tokens_seen": 1088552960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033848545636910735, + "loss": 2.7046, + "theoretical_loss": 3.620049779085573, + "tokens_seen": 1088618496 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033847542627883653, + "loss": 2.8518, + "theoretical_loss": 3.6200293642904824, + "tokens_seen": 1088684032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384653961885657, + "loss": 2.8441, + "theoretical_loss": 3.620008951068347, + "tokens_seen": 1088749568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384553660982949, + "loss": 2.6895, + "theoretical_loss": 3.6199885394189515, + "tokens_seen": 1088815104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384453360080241, + "loss": 2.9414, + "theoretical_loss": 3.619968129342079, + "tokens_seen": 1088880640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384353059177533, + "loss": 2.9459, + "theoretical_loss": 3.619947720837514, + "tokens_seen": 1088946176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033842527582748244, + "loss": 2.7135, + "theoretical_loss": 3.619927313905041, + "tokens_seen": 1089011712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033841524573721167, + "loss": 2.7475, + "theoretical_loss": 3.6199069085444444, + "tokens_seen": 1089077248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384052156469408, + "loss": 2.5889, + "theoretical_loss": 3.6198865047555087, + "tokens_seen": 1089142784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033839518555667003, + "loss": 2.7581, + "theoretical_loss": 3.6198661025380177, + "tokens_seen": 1089208320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383851554663992, + "loss": 2.7403, + "theoretical_loss": 3.619845701891756, + "tokens_seen": 1089273856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383751253761284, + "loss": 2.6924, + "theoretical_loss": 3.6198253028165084, + "tokens_seen": 1089339392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383650952858576, + "loss": 2.7452, + "theoretical_loss": 3.6198049053120593, + "tokens_seen": 1089404928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383550651955868, + "loss": 2.8973, + "theoretical_loss": 3.6197845093781926, + "tokens_seen": 1089470464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 606118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.570420980453491, + "objective/train/theoretical_loss": 3.6197641150146938, + "objective/train/tokens_used": 1109996000, + "theoretical_loss": 3.6197641150146938, + "tokens_seen": 1089536000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033834503510531594, + "loss": 2.6581, + "theoretical_loss": 3.6197641150146938, + "tokens_seen": 1089536000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383350050150452, + "loss": 2.684, + "theoretical_loss": 3.6197437222213473, + "tokens_seen": 1089601536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383249749247743, + "loss": 2.8388, + "theoretical_loss": 3.6197233309979375, + "tokens_seen": 1089667072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033831494483450354, + "loss": 2.7701, + "theoretical_loss": 3.61970294134425, + "tokens_seen": 1089732608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383049147442327, + "loss": 2.7718, + "theoretical_loss": 3.619682553260068, + "tokens_seen": 1089798144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382948846539619, + "loss": 2.6133, + "theoretical_loss": 3.619662166745177, + "tokens_seen": 1089863680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382848545636911, + "loss": 2.8218, + "theoretical_loss": 3.6196417817993627, + "tokens_seen": 1089929216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033827482447342026, + "loss": 2.9036, + "theoretical_loss": 3.6196213984224093, + "tokens_seen": 1089994752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033826479438314944, + "loss": 2.937, + "theoretical_loss": 3.619601016614102, + "tokens_seen": 1090060288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382547642928787, + "loss": 2.7739, + "theoretical_loss": 3.6195806363742253, + "tokens_seen": 1090125824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382447342026078, + "loss": 2.7651, + "theoretical_loss": 3.619560257702564, + "tokens_seen": 1090191360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033823470411233704, + "loss": 2.6242, + "theoretical_loss": 3.6195398805989045, + "tokens_seen": 1090256896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033822467402206617, + "loss": 2.9022, + "theoretical_loss": 3.619519505063031, + "tokens_seen": 1090322432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382146439317954, + "loss": 2.6241, + "theoretical_loss": 3.619499131094728, + "tokens_seen": 1090387968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382046138415246, + "loss": 2.6972, + "theoretical_loss": 3.6194787586937824, + "tokens_seen": 1090453504 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033819458375125376, + "loss": 2.8839, + "theoretical_loss": 3.6194583878599778, + "tokens_seen": 1090519040 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033818455366098295, + "loss": 2.8666, + "theoretical_loss": 3.6194380185931, + "tokens_seen": 1090584576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381745235707122, + "loss": 2.7848, + "theoretical_loss": 3.6194176508929354, + "tokens_seen": 1090650112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381644934804413, + "loss": 2.8002, + "theoretical_loss": 3.619397284759268, + "tokens_seen": 1090715648 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033815446339017054, + "loss": 2.8289, + "theoretical_loss": 3.6193769201918835, + "tokens_seen": 1090781184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033814443329989967, + "loss": 2.8468, + "theoretical_loss": 3.6193565571905673, + "tokens_seen": 1090846720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381344032096289, + "loss": 2.9158, + "theoretical_loss": 3.619336195755105, + "tokens_seen": 1090912256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381243731193581, + "loss": 2.8524, + "theoretical_loss": 3.619315835885283, + "tokens_seen": 1090977792 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033811434302908727, + "loss": 2.6606, + "theoretical_loss": 3.619295477580885, + "tokens_seen": 1091043328 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033810431293881645, + "loss": 3.0112, + "theoretical_loss": 3.6192751208416984, + "tokens_seen": 1091108864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 606665, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6309762001037598, + "objective/train/theoretical_loss": 3.6192547656675083, + "objective/train/tokens_used": 1111634400, + "theoretical_loss": 3.6192547656675083, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033809428284854563, + "loss": 2.6641, + "theoretical_loss": 3.6192547656675083, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380842527582748, + "loss": 2.6976, + "theoretical_loss": 3.6192344120581, + "tokens_seen": 1091239936 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033807422266800405, + "loss": 2.7123, + "theoretical_loss": 3.6192140600132596, + "tokens_seen": 1091305472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380641925777332, + "loss": 2.8793, + "theoretical_loss": 3.619193709532773, + "tokens_seen": 1091371008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380541624874624, + "loss": 2.6522, + "theoretical_loss": 3.6191733606164256, + "tokens_seen": 1091436544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033804413239719154, + "loss": 2.8821, + "theoretical_loss": 3.619153013264003, + "tokens_seen": 1091502080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033803410230692077, + "loss": 2.7992, + "theoretical_loss": 3.6191326674752924, + "tokens_seen": 1091567616 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033802407221664995, + "loss": 2.979, + "theoretical_loss": 3.6191123232500786, + "tokens_seen": 1091633152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033801404212637913, + "loss": 2.7271, + "theoretical_loss": 3.6190919805881485, + "tokens_seen": 1091698688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380040120361083, + "loss": 2.8355, + "theoretical_loss": 3.6190716394892872, + "tokens_seen": 1091764224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033799398194583755, + "loss": 2.7256, + "theoretical_loss": 3.6190512999532807, + "tokens_seen": 1091829760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379839518555667, + "loss": 2.6524, + "theoretical_loss": 3.6190309619799166, + "tokens_seen": 1091895296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379739217652959, + "loss": 2.8708, + "theoretical_loss": 3.619010625568979, + "tokens_seen": 1091960832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033796389167502504, + "loss": 2.7305, + "theoretical_loss": 3.618990290720256, + "tokens_seen": 1092026368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379538615847543, + "loss": 2.7191, + "theoretical_loss": 3.618969957433533, + "tokens_seen": 1092091904 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033794383149448346, + "loss": 2.8005, + "theoretical_loss": 3.618949625708596, + "tokens_seen": 1092157440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033793380140421264, + "loss": 2.7105, + "theoretical_loss": 3.6189292955452315, + "tokens_seen": 1092222976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379237713139418, + "loss": 2.79, + "theoretical_loss": 3.6189089669432266, + "tokens_seen": 1092288512 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337913741223671, + "loss": 3.0197, + "theoretical_loss": 3.6188886399023668, + "tokens_seen": 1092354048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379037111334002, + "loss": 2.8093, + "theoretical_loss": 3.618868314422439, + "tokens_seen": 1092419584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378936810431294, + "loss": 2.9259, + "theoretical_loss": 3.61884799050323, + "tokens_seen": 1092485120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033788365095285854, + "loss": 2.807, + "theoretical_loss": 3.6188276681445255, + "tokens_seen": 1092550656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378736208625878, + "loss": 2.4255, + "theoretical_loss": 3.6188073473461126, + "tokens_seen": 1092616192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378635907723169, + "loss": 2.679, + "theoretical_loss": 3.6187870281077785, + "tokens_seen": 1092681728 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033785356068204614, + "loss": 2.7072, + "theoretical_loss": 3.618766710429308, + "tokens_seen": 1092747264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 607551, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.578968048095703, + "objective/train/theoretical_loss": 3.6187463943104903, + "objective/train/tokens_used": 1113272800, + "theoretical_loss": 3.6187463943104903, + "tokens_seen": 1092812800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378435305917753, + "loss": 2.717, + "theoretical_loss": 3.6187463943104903, + "tokens_seen": 1092812800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378335005015045, + "loss": 2.8585, + "theoretical_loss": 3.6187260797511103, + "tokens_seen": 1092878336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033782347041123374, + "loss": 2.6637, + "theoretical_loss": 3.6187057667509555, + "tokens_seen": 1092943872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378134403209629, + "loss": 2.915, + "theoretical_loss": 3.618685455309813, + "tokens_seen": 1093009408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378034102306921, + "loss": 2.9441, + "theoretical_loss": 3.618665145427469, + "tokens_seen": 1093074944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377933801404213, + "loss": 2.9739, + "theoretical_loss": 3.6186448371037105, + "tokens_seen": 1093140480 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033778335005015046, + "loss": 2.8502, + "theoretical_loss": 3.6186245303383253, + "tokens_seen": 1093206016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033777331995987964, + "loss": 2.7294, + "theoretical_loss": 3.6186042251311, + "tokens_seen": 1093271552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377632898696089, + "loss": 2.8688, + "theoretical_loss": 3.6185839214818207, + "tokens_seen": 1093337088 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337753259779338, + "loss": 3.0609, + "theoretical_loss": 3.618563619390276, + "tokens_seen": 1093402624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033774322968906724, + "loss": 2.6654, + "theoretical_loss": 3.6185433188562515, + "tokens_seen": 1093468160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033773319959879637, + "loss": 2.6188, + "theoretical_loss": 3.618523019879536, + "tokens_seen": 1093533696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377231695085256, + "loss": 2.814, + "theoretical_loss": 3.6185027224599153, + "tokens_seen": 1093599232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377131394182548, + "loss": 2.7198, + "theoretical_loss": 3.6184824265971773, + "tokens_seen": 1093664768 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033770310932798396, + "loss": 2.6649, + "theoretical_loss": 3.6184621322911097, + "tokens_seen": 1093730304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033769307923771315, + "loss": 2.8431, + "theoretical_loss": 3.618441839541499, + "tokens_seen": 1093795840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376830491474424, + "loss": 2.9257, + "theoretical_loss": 3.618421548348133, + "tokens_seen": 1093861376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376730190571715, + "loss": 2.6722, + "theoretical_loss": 3.618401258710799, + "tokens_seen": 1093926912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033766298896690074, + "loss": 2.7113, + "theoretical_loss": 3.6183809706292847, + "tokens_seen": 1093992448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033765295887662987, + "loss": 2.6744, + "theoretical_loss": 3.6183606841033775, + "tokens_seen": 1094057984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376429287863591, + "loss": 2.9785, + "theoretical_loss": 3.618340399132865, + "tokens_seen": 1094123520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376328986960883, + "loss": 2.7727, + "theoretical_loss": 3.6183201157175344, + "tokens_seen": 1094189056 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033762286860581747, + "loss": 2.9491, + "theoretical_loss": 3.618299833857174, + "tokens_seen": 1094254592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033761283851554665, + "loss": 2.6171, + "theoretical_loss": 3.6182795535515706, + "tokens_seen": 1094320128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033760280842527583, + "loss": 3.0635, + "theoretical_loss": 3.618259274800513, + "tokens_seen": 1094385664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 607859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.727546453475952, + "objective/train/theoretical_loss": 3.618238997603788, + "objective/train/tokens_used": 1114911200, + "theoretical_loss": 3.618238997603788, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337592778335005, + "loss": 2.8344, + "theoretical_loss": 3.618238997603788, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033758274824473425, + "loss": 2.8925, + "theoretical_loss": 3.618218721961184, + "tokens_seen": 1094516736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375727181544634, + "loss": 2.9069, + "theoretical_loss": 3.618198447872489, + "tokens_seen": 1094582272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375626880641926, + "loss": 2.8822, + "theoretical_loss": 3.6181781753374898, + "tokens_seen": 1094647808 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033755265797392174, + "loss": 2.8686, + "theoretical_loss": 3.618157904355975, + "tokens_seen": 1094713344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033754262788365097, + "loss": 2.5076, + "theoretical_loss": 3.6181376349277334, + "tokens_seen": 1094778880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033753259779338015, + "loss": 2.857, + "theoretical_loss": 3.6181173670525517, + "tokens_seen": 1094844416 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033752256770310933, + "loss": 2.6923, + "theoretical_loss": 3.6180971007302185, + "tokens_seen": 1094909952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375125376128385, + "loss": 2.4682, + "theoretical_loss": 3.6180768359605215, + "tokens_seen": 1094975488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033750250752256775, + "loss": 2.9568, + "theoretical_loss": 3.61805657274325, + "tokens_seen": 1095041024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374924774322969, + "loss": 2.7654, + "theoretical_loss": 3.618036311078191, + "tokens_seen": 1095106560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374824473420261, + "loss": 2.8227, + "theoretical_loss": 3.618016050965133, + "tokens_seen": 1095172096 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033747241725175524, + "loss": 2.7536, + "theoretical_loss": 3.617995792403864, + "tokens_seen": 1095237632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374623871614845, + "loss": 2.5663, + "theoretical_loss": 3.617975535394173, + "tokens_seen": 1095303168 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033745235707121366, + "loss": 2.8911, + "theoretical_loss": 3.617955279935848, + "tokens_seen": 1095368704 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033744232698094284, + "loss": 2.5522, + "theoretical_loss": 3.617935026028677, + "tokens_seen": 1095434240 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337432296890672, + "loss": 2.6732, + "theoretical_loss": 3.6179147736724495, + "tokens_seen": 1095499776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374222668004012, + "loss": 2.9258, + "theoretical_loss": 3.617894522866953, + "tokens_seen": 1095565312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374122367101304, + "loss": 2.6745, + "theoretical_loss": 3.6178742736119758, + "tokens_seen": 1095630848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374022066198596, + "loss": 2.4884, + "theoretical_loss": 3.617854025907307, + "tokens_seen": 1095696384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033739217652958874, + "loss": 2.5483, + "theoretical_loss": 3.6178337797527353, + "tokens_seen": 1095761920 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337382146439318, + "loss": 2.7345, + "theoretical_loss": 3.6178135351480494, + "tokens_seen": 1095827456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373721163490471, + "loss": 2.825, + "theoretical_loss": 3.617793292093037, + "tokens_seen": 1095892992 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033736208625877634, + "loss": 2.8887, + "theoretical_loss": 3.6177730505874885, + "tokens_seen": 1095958528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373520561685055, + "loss": 2.8044, + "theoretical_loss": 3.617752810631191, + "tokens_seen": 1096024064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 608889, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.800088882446289, + "objective/train/theoretical_loss": 3.6177325722239337, + "objective/train/tokens_used": 1116549600, + "theoretical_loss": 3.6177325722239337, + "tokens_seen": 1096089600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373420260782347, + "loss": 2.8135, + "theoretical_loss": 3.6177325722239337, + "tokens_seen": 1096089600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373319959879639, + "loss": 2.7077, + "theoretical_loss": 3.617712335365506, + "tokens_seen": 1096155136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373219658976931, + "loss": 2.7595, + "theoretical_loss": 3.6176921000556965, + "tokens_seen": 1096220672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033731193580742225, + "loss": 2.9444, + "theoretical_loss": 3.617671866294294, + "tokens_seen": 1096286208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373019057171515, + "loss": 2.6648, + "theoretical_loss": 3.6176516340810876, + "tokens_seen": 1096351744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372918756268806, + "loss": 2.7699, + "theoretical_loss": 3.6176314034158663, + "tokens_seen": 1096417280 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033728184553660984, + "loss": 2.8946, + "theoretical_loss": 3.6176111742984194, + "tokens_seen": 1096482816 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337271815446339, + "loss": 2.7141, + "theoretical_loss": 3.6175909467285354, + "tokens_seen": 1096548352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372617853560682, + "loss": 2.8175, + "theoretical_loss": 3.617570720706004, + "tokens_seen": 1096613888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372517552657974, + "loss": 2.9539, + "theoretical_loss": 3.6175504962306135, + "tokens_seen": 1096679424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033724172517552657, + "loss": 2.7993, + "theoretical_loss": 3.6175302733021546, + "tokens_seen": 1096744960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033723169508525575, + "loss": 2.8731, + "theoretical_loss": 3.617510051920415, + "tokens_seen": 1096810496 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337221664994985, + "loss": 2.6325, + "theoretical_loss": 3.6174898320851847, + "tokens_seen": 1096876032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372116349047141, + "loss": 2.826, + "theoretical_loss": 3.617469613796253, + "tokens_seen": 1096941568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033720160481444335, + "loss": 2.8224, + "theoretical_loss": 3.6174493970534094, + "tokens_seen": 1097007104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371915747241725, + "loss": 2.8515, + "theoretical_loss": 3.6174291818564432, + "tokens_seen": 1097072640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371815446339017, + "loss": 2.7638, + "theoretical_loss": 3.617408968205144, + "tokens_seen": 1097138176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371715145436309, + "loss": 2.8324, + "theoretical_loss": 3.6173887560993006, + "tokens_seen": 1097203712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033716148445336007, + "loss": 2.891, + "theoretical_loss": 3.6173685455387035, + "tokens_seen": 1097269248 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033715145436308925, + "loss": 2.8952, + "theoretical_loss": 3.6173483365231416, + "tokens_seen": 1097334784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371414242728185, + "loss": 2.7169, + "theoretical_loss": 3.6173281290524053, + "tokens_seen": 1097400320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371313941825476, + "loss": 2.8957, + "theoretical_loss": 3.6173079231262832, + "tokens_seen": 1097465856 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033712136409227685, + "loss": 2.7076, + "theoretical_loss": 3.6172877187445662, + "tokens_seen": 1097531392 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337111334002006, + "loss": 2.7603, + "theoretical_loss": 3.6172675159070424, + "tokens_seen": 1097596928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371013039117352, + "loss": 2.7749, + "theoretical_loss": 3.617247314613503, + "tokens_seen": 1097662464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 610057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0604283809661865, + "objective/train/theoretical_loss": 3.617227114863738, + "objective/train/tokens_used": 1118188000, + "theoretical_loss": 3.617227114863738, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370912738214644, + "loss": 3.0146, + "theoretical_loss": 3.617227114863738, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370812437311936, + "loss": 2.8418, + "theoretical_loss": 3.617206916657536, + "tokens_seen": 1097793536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370712136409228, + "loss": 2.7766, + "theoretical_loss": 3.617186719994688, + "tokens_seen": 1097859072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033706118355065194, + "loss": 2.7925, + "theoretical_loss": 3.6171665248749836, + "tokens_seen": 1097924608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033705115346038117, + "loss": 2.6988, + "theoretical_loss": 3.6171463312982124, + "tokens_seen": 1097990144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033704112337011035, + "loss": 2.9721, + "theoretical_loss": 3.617126139264165, + "tokens_seen": 1098055680 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033703109327983953, + "loss": 2.8138, + "theoretical_loss": 3.617105948772631, + "tokens_seen": 1098121216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370210631895687, + "loss": 2.5797, + "theoretical_loss": 3.617085759823401, + "tokens_seen": 1098186752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033701103309929795, + "loss": 2.7429, + "theoretical_loss": 3.6170655724162653, + "tokens_seen": 1098252288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370010030090271, + "loss": 2.6726, + "theoretical_loss": 3.6170453865510135, + "tokens_seen": 1098317824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369909729187563, + "loss": 2.8859, + "theoretical_loss": 3.6170252022274356, + "tokens_seen": 1098383360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033698094282848544, + "loss": 2.7337, + "theoretical_loss": 3.6170050194453225, + "tokens_seen": 1098448896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369709127382147, + "loss": 2.7052, + "theoretical_loss": 3.616984838204465, + "tokens_seen": 1098514432 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033696088264794386, + "loss": 2.6465, + "theoretical_loss": 3.616964658504652, + "tokens_seen": 1098579968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033695085255767304, + "loss": 2.5825, + "theoretical_loss": 3.6169444803456754, + "tokens_seen": 1098645504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369408224674022, + "loss": 2.7459, + "theoretical_loss": 3.616924303727325, + "tokens_seen": 1098711040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369307923771314, + "loss": 2.722, + "theoretical_loss": 3.616904128649391, + "tokens_seen": 1098776576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369207622868606, + "loss": 2.6449, + "theoretical_loss": 3.616883955111664, + "tokens_seen": 1098842112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369107321965898, + "loss": 2.7865, + "theoretical_loss": 3.616863783113935, + "tokens_seen": 1098907648 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033690070210631894, + "loss": 2.7845, + "theoretical_loss": 3.6168436126559946, + "tokens_seen": 1098973184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368906720160482, + "loss": 2.5238, + "theoretical_loss": 3.616823443737633, + "tokens_seen": 1099038720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368806419257773, + "loss": 2.7368, + "theoretical_loss": 3.616803276358641, + "tokens_seen": 1099104256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033687061183550654, + "loss": 2.6779, + "theoretical_loss": 3.61678311051881, + "tokens_seen": 1099169792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368605817452357, + "loss": 2.7011, + "theoretical_loss": 3.61676294621793, + "tokens_seen": 1099235328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368505516549649, + "loss": 2.7203, + "theoretical_loss": 3.6167427834557913, + "tokens_seen": 1099300864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 610651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6474804878234863, + "objective/train/theoretical_loss": 3.6167226222321864, + "objective/train/tokens_used": 1119826400, + "theoretical_loss": 3.6167226222321864, + "tokens_seen": 1099366400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368405215646941, + "loss": 2.5971, + "theoretical_loss": 3.6167226222321864, + "tokens_seen": 1099366400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368304914744233, + "loss": 2.7681, + "theoretical_loss": 3.616702462546905, + "tokens_seen": 1099431936 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033682046138415245, + "loss": 2.737, + "theoretical_loss": 3.6166823043997383, + "tokens_seen": 1099497472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368104312938817, + "loss": 2.7658, + "theoretical_loss": 3.6166621477904775, + "tokens_seen": 1099563008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368004012036108, + "loss": 2.4798, + "theoretical_loss": 3.6166419927189133, + "tokens_seen": 1099628544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033679037111334004, + "loss": 2.4484, + "theoretical_loss": 3.6166218391848366, + "tokens_seen": 1099694080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367803410230692, + "loss": 2.7835, + "theoretical_loss": 3.616601687188039, + "tokens_seen": 1099759616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367703109327984, + "loss": 2.6162, + "theoretical_loss": 3.6165815367283116, + "tokens_seen": 1099825152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367602808425276, + "loss": 2.8223, + "theoretical_loss": 3.6165613878054454, + "tokens_seen": 1099890688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033675025075225677, + "loss": 2.8348, + "theoretical_loss": 3.616541240419232, + "tokens_seen": 1099956224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033674022066198595, + "loss": 2.9311, + "theoretical_loss": 3.6165210945694612, + "tokens_seen": 1100021760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367301905717152, + "loss": 2.9308, + "theoretical_loss": 3.616500950255926, + "tokens_seen": 1100087296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367201604814443, + "loss": 2.5941, + "theoretical_loss": 3.6164808074784176, + "tokens_seen": 1100152832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033671013039117355, + "loss": 2.7612, + "theoretical_loss": 3.6164606662367262, + "tokens_seen": 1100218368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033670010030090273, + "loss": 2.7321, + "theoretical_loss": 3.6164405265306447, + "tokens_seen": 1100283904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366900702106319, + "loss": 2.6555, + "theoretical_loss": 3.6164203883599635, + "tokens_seen": 1100349440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366800401203611, + "loss": 2.7248, + "theoretical_loss": 3.6164002517244738, + "tokens_seen": 1100414976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033667001003009027, + "loss": 2.9274, + "theoretical_loss": 3.6163801166239686, + "tokens_seen": 1100480512 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033665997993981945, + "loss": 2.6037, + "theoretical_loss": 3.6163599830582385, + "tokens_seen": 1100546048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366499498495487, + "loss": 2.6833, + "theoretical_loss": 3.6163398510270754, + "tokens_seen": 1100611584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366399197592778, + "loss": 2.9599, + "theoretical_loss": 3.6163197205302704, + "tokens_seen": 1100677120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033662988966900705, + "loss": 2.6235, + "theoretical_loss": 3.616299591567616, + "tokens_seen": 1100742656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366198595787362, + "loss": 2.6697, + "theoretical_loss": 3.6162794641389038, + "tokens_seen": 1100808192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366098294884654, + "loss": 2.764, + "theoretical_loss": 3.616259338243925, + "tokens_seen": 1100873728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365997993981946, + "loss": 2.7095, + "theoretical_loss": 3.6162392138824724, + "tokens_seen": 1100939264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 611659, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3433709144592285, + "objective/train/theoretical_loss": 3.6162190910543366, + "objective/train/tokens_used": 1121464800, + "theoretical_loss": 3.6162190910543366, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365897693079238, + "loss": 2.5755, + "theoretical_loss": 3.6162190910543366, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033657973921765295, + "loss": 2.8004, + "theoretical_loss": 3.616198969759311, + "tokens_seen": 1101070336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033656970912738214, + "loss": 2.7259, + "theoretical_loss": 3.6161788499971865, + "tokens_seen": 1101135872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365596790371113, + "loss": 2.8029, + "theoretical_loss": 3.6161587317677553, + "tokens_seen": 1101201408 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033654964894684055, + "loss": 2.6944, + "theoretical_loss": 3.6161386150708097, + "tokens_seen": 1101266944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365396188565697, + "loss": 2.7386, + "theoretical_loss": 3.6161184999061415, + "tokens_seen": 1101332480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365295887662989, + "loss": 2.6169, + "theoretical_loss": 3.616098386273543, + "tokens_seen": 1101398016 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365195586760281, + "loss": 2.9178, + "theoretical_loss": 3.6160782741728066, + "tokens_seen": 1101463552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365095285857573, + "loss": 2.8122, + "theoretical_loss": 3.616058163603724, + "tokens_seen": 1101529088 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033649949849548646, + "loss": 2.6384, + "theoretical_loss": 3.6160380545660873, + "tokens_seen": 1101594624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033648946840521564, + "loss": 2.9066, + "theoretical_loss": 3.6160179470596896, + "tokens_seen": 1101660160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364794383149448, + "loss": 2.8531, + "theoretical_loss": 3.6159978410843223, + "tokens_seen": 1101725696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033646940822467406, + "loss": 2.5062, + "theoretical_loss": 3.6159777366397785, + "tokens_seen": 1101791232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364593781344032, + "loss": 2.7402, + "theoretical_loss": 3.6159576337258503, + "tokens_seen": 1101856768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364493480441324, + "loss": 2.8608, + "theoretical_loss": 3.61593753234233, + "tokens_seen": 1101922304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033643931795386154, + "loss": 2.7708, + "theoretical_loss": 3.6159174324890104, + "tokens_seen": 1101987840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364292878635908, + "loss": 3.0905, + "theoretical_loss": 3.6158973341656835, + "tokens_seen": 1102053376 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033641925777331996, + "loss": 2.9452, + "theoretical_loss": 3.6158772373721426, + "tokens_seen": 1102118912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033640922768304914, + "loss": 2.8754, + "theoretical_loss": 3.6158571421081795, + "tokens_seen": 1102184448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363991975927783, + "loss": 2.7918, + "theoretical_loss": 3.6158370483735878, + "tokens_seen": 1102249984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363891675025075, + "loss": 3.0109, + "theoretical_loss": 3.6158169561681595, + "tokens_seen": 1102315520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363791374122367, + "loss": 2.9653, + "theoretical_loss": 3.6157968654916868, + "tokens_seen": 1102381056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363691073219659, + "loss": 2.9488, + "theoretical_loss": 3.6157767763439637, + "tokens_seen": 1102446592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033635907723169505, + "loss": 2.751, + "theoretical_loss": 3.6157566887247823, + "tokens_seen": 1102512128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363490471414243, + "loss": 2.9518, + "theoretical_loss": 3.6157366026339357, + "tokens_seen": 1102577664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 612217, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9327733516693115, + "objective/train/theoretical_loss": 3.615716518071217, + "objective/train/tokens_used": 1123103200, + "theoretical_loss": 3.615716518071217, + "tokens_seen": 1102643200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033633901705115346, + "loss": 2.9102, + "theoretical_loss": 3.615716518071217, + "tokens_seen": 1102643200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033632898696088265, + "loss": 2.7885, + "theoretical_loss": 3.6156964350364182, + "tokens_seen": 1102708736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363189568706119, + "loss": 2.7765, + "theoretical_loss": 3.615676353529333, + "tokens_seen": 1102774272 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336308926780341, + "loss": 2.9011, + "theoretical_loss": 3.6156562735497544, + "tokens_seen": 1102839808 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033629889669007024, + "loss": 2.8588, + "theoretical_loss": 3.6156361950974754, + "tokens_seen": 1102905344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362888665997994, + "loss": 2.6921, + "theoretical_loss": 3.615616118172289, + "tokens_seen": 1102970880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362788365095286, + "loss": 2.725, + "theoretical_loss": 3.615596042773988, + "tokens_seen": 1103036416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362688064192578, + "loss": 2.8121, + "theoretical_loss": 3.615575968902366, + "tokens_seen": 1103101952 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033625877632898697, + "loss": 2.745, + "theoretical_loss": 3.6155558965572165, + "tokens_seen": 1103167488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033624874623871615, + "loss": 2.7481, + "theoretical_loss": 3.6155358257383323, + "tokens_seen": 1103233024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362387161484454, + "loss": 2.7961, + "theoretical_loss": 3.6155157564455065, + "tokens_seen": 1103298560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362286860581745, + "loss": 2.536, + "theoretical_loss": 3.615495688678533, + "tokens_seen": 1103364096 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033621865596790375, + "loss": 2.5681, + "theoretical_loss": 3.6154756224372044, + "tokens_seen": 1103429632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033620862587763293, + "loss": 2.7488, + "theoretical_loss": 3.615455557721315, + "tokens_seen": 1103495168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361985957873621, + "loss": 2.7733, + "theoretical_loss": 3.615435494530658, + "tokens_seen": 1103560704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361885656970913, + "loss": 2.7637, + "theoretical_loss": 3.615415432865026, + "tokens_seen": 1103626240 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033617853560682047, + "loss": 2.5859, + "theoretical_loss": 3.6153953727242136, + "tokens_seen": 1103691776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033616850551654965, + "loss": 3.0173, + "theoretical_loss": 3.6153753141080145, + "tokens_seen": 1103757312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361584754262789, + "loss": 2.6315, + "theoretical_loss": 3.615355257016221, + "tokens_seen": 1103822848 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336148445336008, + "loss": 2.6973, + "theoretical_loss": 3.615335201448628, + "tokens_seen": 1103888384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033613841524573725, + "loss": 2.8397, + "theoretical_loss": 3.6153151474050285, + "tokens_seen": 1103953920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361283851554664, + "loss": 3.0104, + "theoretical_loss": 3.6152950948852167, + "tokens_seen": 1104019456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361183550651956, + "loss": 2.8473, + "theoretical_loss": 3.6152750438889862, + "tokens_seen": 1104084992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361083249749248, + "loss": 2.7402, + "theoretical_loss": 3.6152549944161314, + "tokens_seen": 1104150528 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336098294884654, + "loss": 2.5521, + "theoretical_loss": 3.615234946466445, + "tokens_seen": 1104216064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 613498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7340283393859863, + "objective/train/theoretical_loss": 3.615214900039721, + "objective/train/tokens_used": 1124741600, + "theoretical_loss": 3.615214900039721, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033608826479438316, + "loss": 2.6072, + "theoretical_loss": 3.615214900039721, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033607823470411234, + "loss": 2.713, + "theoretical_loss": 3.6151948551357544, + "tokens_seen": 1104347136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360682046138415, + "loss": 2.7713, + "theoretical_loss": 3.6151748117543385, + "tokens_seen": 1104412672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033605817452357075, + "loss": 2.7701, + "theoretical_loss": 3.615154769895267, + "tokens_seen": 1104478208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360481444332999, + "loss": 2.7249, + "theoretical_loss": 3.6151347295583345, + "tokens_seen": 1104543744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360381143430291, + "loss": 2.7956, + "theoretical_loss": 3.615114690743335, + "tokens_seen": 1104609280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360280842527583, + "loss": 2.9324, + "theoretical_loss": 3.6150946534500625, + "tokens_seen": 1104674816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360180541624875, + "loss": 2.7668, + "theoretical_loss": 3.615074617678311, + "tokens_seen": 1104740352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033600802407221666, + "loss": 2.775, + "theoretical_loss": 3.6150545834278756, + "tokens_seen": 1104805888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033599799398194584, + "loss": 2.7627, + "theoretical_loss": 3.615034550698549, + "tokens_seen": 1104871424 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335987963891675, + "loss": 2.7189, + "theoretical_loss": 3.615014519490127, + "tokens_seen": 1104936960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033597793380140426, + "loss": 2.8479, + "theoretical_loss": 3.6149944898024033, + "tokens_seen": 1105002496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359679037111334, + "loss": 2.7308, + "theoretical_loss": 3.614974461635172, + "tokens_seen": 1105068032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359578736208626, + "loss": 2.742, + "theoretical_loss": 3.6149544349882277, + "tokens_seen": 1105133568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033594784353059174, + "loss": 2.8406, + "theoretical_loss": 3.614934409861365, + "tokens_seen": 1105199104 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335937813440321, + "loss": 2.8199, + "theoretical_loss": 3.6149143862543784, + "tokens_seen": 1105264640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033592778335005016, + "loss": 2.737, + "theoretical_loss": 3.614894364167063, + "tokens_seen": 1105330176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033591775325977934, + "loss": 2.8659, + "theoretical_loss": 3.614874343599212, + "tokens_seen": 1105395712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359077231695085, + "loss": 2.751, + "theoretical_loss": 3.614854324550621, + "tokens_seen": 1105461248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358976930792377, + "loss": 2.978, + "theoretical_loss": 3.614834307021084, + "tokens_seen": 1105526784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358876629889669, + "loss": 2.6978, + "theoretical_loss": 3.614814291010397, + "tokens_seen": 1105592320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358776328986961, + "loss": 2.9026, + "theoretical_loss": 3.6147942765183534, + "tokens_seen": 1105657856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033586760280842525, + "loss": 2.7393, + "theoretical_loss": 3.6147742635447484, + "tokens_seen": 1105723392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358575727181545, + "loss": 2.8325, + "theoretical_loss": 3.614754252089377, + "tokens_seen": 1105788928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033584754262788366, + "loss": 3.0232, + "theoretical_loss": 3.614734242152034, + "tokens_seen": 1105854464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 614567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4206838607788086, + "objective/train/theoretical_loss": 3.6147142337325135, + "objective/train/tokens_used": 1126380000, + "theoretical_loss": 3.6147142337325135, + "tokens_seen": 1105920000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033583751253761285, + "loss": 2.6926, + "theoretical_loss": 3.6147142337325135, + "tokens_seen": 1105920000 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335827482447342, + "loss": 2.8486, + "theoretical_loss": 3.6146942268306117, + "tokens_seen": 1105985536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358174523570712, + "loss": 2.7974, + "theoretical_loss": 3.6146742214461227, + "tokens_seen": 1106051072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358074222668004, + "loss": 2.7092, + "theoretical_loss": 3.614654217578842, + "tokens_seen": 1106116608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357973921765296, + "loss": 2.5362, + "theoretical_loss": 3.6146342152285644, + "tokens_seen": 1106182144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033578736208625875, + "loss": 2.8508, + "theoretical_loss": 3.614614214395085, + "tokens_seen": 1106247680 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335777331995988, + "loss": 2.4857, + "theoretical_loss": 3.614594215078199, + "tokens_seen": 1106313216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357673019057171, + "loss": 2.8308, + "theoretical_loss": 3.614574217277702, + "tokens_seen": 1106378752 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033575727181544635, + "loss": 2.8693, + "theoretical_loss": 3.614554220993388, + "tokens_seen": 1106444288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033574724172517553, + "loss": 2.7059, + "theoretical_loss": 3.6145342262250533, + "tokens_seen": 1106509824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357372116349047, + "loss": 2.7467, + "theoretical_loss": 3.6145142329724926, + "tokens_seen": 1106575360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357271815446339, + "loss": 2.6414, + "theoretical_loss": 3.6144942412355023, + "tokens_seen": 1106640896 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033571715145436313, + "loss": 2.8767, + "theoretical_loss": 3.6144742510138768, + "tokens_seen": 1106706432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033570712136409225, + "loss": 2.8899, + "theoretical_loss": 3.614454262307411, + "tokens_seen": 1106771968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356970912738215, + "loss": 2.8324, + "theoretical_loss": 3.6144342751159018, + "tokens_seen": 1106837504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356870611835506, + "loss": 2.5821, + "theoretical_loss": 3.614414289439144, + "tokens_seen": 1106903040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033567703109327985, + "loss": 2.6711, + "theoretical_loss": 3.6143943052769325, + "tokens_seen": 1106968576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033566700100300903, + "loss": 2.7409, + "theoretical_loss": 3.6143743226290637, + "tokens_seen": 1107034112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356569709127382, + "loss": 2.8463, + "theoretical_loss": 3.614354341495333, + "tokens_seen": 1107099648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356469408224674, + "loss": 2.7817, + "theoretical_loss": 3.614334361875536, + "tokens_seen": 1107165184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356369107321966, + "loss": 2.7057, + "theoretical_loss": 3.614314383769469, + "tokens_seen": 1107230720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033562688064192576, + "loss": 2.9353, + "theoretical_loss": 3.6142944071769265, + "tokens_seen": 1107296256 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335616850551655, + "loss": 2.4972, + "theoretical_loss": 3.614274432097705, + "tokens_seen": 1107361792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356068204613841, + "loss": 2.7738, + "theoretical_loss": 3.6142544585316, + "tokens_seen": 1107427328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033559679037111336, + "loss": 2.7836, + "theoretical_loss": 3.614234486478408, + "tokens_seen": 1107492864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 615343, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.606733798980713, + "objective/train/theoretical_loss": 3.614214515937924, + "objective/train/tokens_used": 1128018400, + "theoretical_loss": 3.614214515937924, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355867602808425, + "loss": 2.9232, + "theoretical_loss": 3.614214515937924, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355767301905717, + "loss": 2.8684, + "theoretical_loss": 3.614194546909945, + "tokens_seen": 1107623936 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033556670010030095, + "loss": 2.6234, + "theoretical_loss": 3.6141745793942657, + "tokens_seen": 1107689472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355566700100301, + "loss": 2.6494, + "theoretical_loss": 3.6141546133906832, + "tokens_seen": 1107755008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355466399197593, + "loss": 2.9153, + "theoretical_loss": 3.6141346488989923, + "tokens_seen": 1107820544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355366098294885, + "loss": 2.7833, + "theoretical_loss": 3.6141146859189908, + "tokens_seen": 1107886080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355265797392177, + "loss": 2.5932, + "theoretical_loss": 3.6140947244504735, + "tokens_seen": 1107951616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033551654964894686, + "loss": 2.6819, + "theoretical_loss": 3.614074764493237, + "tokens_seen": 1108017152 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033550651955867604, + "loss": 2.9393, + "theoretical_loss": 3.6140548060470774, + "tokens_seen": 1108082688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354964894684052, + "loss": 3.0045, + "theoretical_loss": 3.614034849111791, + "tokens_seen": 1108148224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033548645937813446, + "loss": 2.7504, + "theoretical_loss": 3.614014893687174, + "tokens_seen": 1108213760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354764292878636, + "loss": 2.8945, + "theoretical_loss": 3.613994939773023, + "tokens_seen": 1108279296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354663991975928, + "loss": 2.7427, + "theoretical_loss": 3.613974987369134, + "tokens_seen": 1108344832 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033545636910732195, + "loss": 2.9324, + "theoretical_loss": 3.613955036475304, + "tokens_seen": 1108410368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354463390170512, + "loss": 2.575, + "theoretical_loss": 3.6139350870913285, + "tokens_seen": 1108475904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033543630892678036, + "loss": 2.6235, + "theoretical_loss": 3.6139151392170046, + "tokens_seen": 1108541440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033542627883650954, + "loss": 2.7408, + "theoretical_loss": 3.613895192852129, + "tokens_seen": 1108606976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354162487462387, + "loss": 2.7964, + "theoretical_loss": 3.6138752479964973, + "tokens_seen": 1108672512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354062186559679, + "loss": 2.8134, + "theoretical_loss": 3.6138553046499076, + "tokens_seen": 1108738048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353961885656971, + "loss": 2.683, + "theoretical_loss": 3.6138353628121553, + "tokens_seen": 1108803584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353861584754263, + "loss": 2.4956, + "theoretical_loss": 3.6138154224830377, + "tokens_seen": 1108869120 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033537612838515545, + "loss": 2.766, + "theoretical_loss": 3.6137954836623507, + "tokens_seen": 1108934656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353660982948847, + "loss": 2.6687, + "theoretical_loss": 3.613775546349892, + "tokens_seen": 1109000192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033535606820461386, + "loss": 2.9337, + "theoretical_loss": 3.613755610545458, + "tokens_seen": 1109065728 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033534603811434305, + "loss": 2.6042, + "theoretical_loss": 3.6137356762488464, + "tokens_seen": 1109131264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 616276, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3585472106933594, + "objective/train/theoretical_loss": 3.613715743459852, + "objective/train/tokens_used": 1129656800, + "theoretical_loss": 3.613715743459852, + "tokens_seen": 1109196800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353360080240722, + "loss": 3.0601, + "theoretical_loss": 3.613715743459852, + "tokens_seen": 1109196800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353259779338014, + "loss": 2.673, + "theoretical_loss": 3.613695812178274, + "tokens_seen": 1109262336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353159478435306, + "loss": 2.7332, + "theoretical_loss": 3.613675882403908, + "tokens_seen": 1109327872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353059177532598, + "loss": 2.997, + "theoretical_loss": 3.6136559541365507, + "tokens_seen": 1109393408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033529588766298895, + "loss": 2.732, + "theoretical_loss": 3.6136360273760006, + "tokens_seen": 1109458944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352858575727182, + "loss": 2.7252, + "theoretical_loss": 3.6136161021220534, + "tokens_seen": 1109524480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352758274824473, + "loss": 2.8383, + "theoretical_loss": 3.6135961783745065, + "tokens_seen": 1109590016 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033526579739217655, + "loss": 2.836, + "theoretical_loss": 3.613576256133158, + "tokens_seen": 1109655552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033525576730190573, + "loss": 2.694, + "theoretical_loss": 3.6135563353978037, + "tokens_seen": 1109721088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352457372116349, + "loss": 2.6563, + "theoretical_loss": 3.613536416168242, + "tokens_seen": 1109786624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352357071213641, + "loss": 2.497, + "theoretical_loss": 3.6135164984442696, + "tokens_seen": 1109852160 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033522567703109333, + "loss": 2.6525, + "theoretical_loss": 3.6134965822256837, + "tokens_seen": 1109917696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033521564694082245, + "loss": 2.7866, + "theoretical_loss": 3.613476667512282, + "tokens_seen": 1109983232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352056168505517, + "loss": 2.7501, + "theoretical_loss": 3.6134567543038614, + "tokens_seen": 1110048768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351955867602808, + "loss": 2.7559, + "theoretical_loss": 3.61343684260022, + "tokens_seen": 1110114304 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033518555667001005, + "loss": 2.8027, + "theoretical_loss": 3.6134169324011545, + "tokens_seen": 1110179840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033517552657973923, + "loss": 2.7998, + "theoretical_loss": 3.613397023706463, + "tokens_seen": 1110245376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351654964894684, + "loss": 2.9335, + "theoretical_loss": 3.6133771165159425, + "tokens_seen": 1110310912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351554663991976, + "loss": 2.8296, + "theoretical_loss": 3.6133572108293914, + "tokens_seen": 1110376448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351454363089268, + "loss": 2.7763, + "theoretical_loss": 3.613337306646607, + "tokens_seen": 1110441984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033513540621865596, + "loss": 2.7825, + "theoretical_loss": 3.613317403967386, + "tokens_seen": 1110507520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351253761283852, + "loss": 3.0032, + "theoretical_loss": 3.6132975027915277, + "tokens_seen": 1110573056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351153460381143, + "loss": 2.4999, + "theoretical_loss": 3.6132776031188287, + "tokens_seen": 1110638592 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033510531594784356, + "loss": 2.7841, + "theoretical_loss": 3.6132577049490866, + "tokens_seen": 1110704128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350952858575727, + "loss": 2.7022, + "theoretical_loss": 3.6132378082821006, + "tokens_seen": 1110769664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 616833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7672462463378906, + "objective/train/theoretical_loss": 3.613217913117667, + "objective/train/tokens_used": 1131295200, + "theoretical_loss": 3.613217913117667, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350852557673019, + "loss": 2.8862, + "theoretical_loss": 3.613217913117667, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350752256770311, + "loss": 3.0087, + "theoretical_loss": 3.6131980194555844, + "tokens_seen": 1110900736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350651955867603, + "loss": 2.8562, + "theoretical_loss": 3.6131781272956514, + "tokens_seen": 1110966272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033505516549648946, + "loss": 2.8075, + "theoretical_loss": 3.6131582366376644, + "tokens_seen": 1111031808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350451354062187, + "loss": 2.7964, + "theoretical_loss": 3.613138347481423, + "tokens_seen": 1111097344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350351053159478, + "loss": 2.6795, + "theoretical_loss": 3.6131184598267243, + "tokens_seen": 1111162880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033502507522567706, + "loss": 2.5942, + "theoretical_loss": 3.6130985736733665, + "tokens_seen": 1111228416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350150451354062, + "loss": 2.6254, + "theoretical_loss": 3.613078689021148, + "tokens_seen": 1111293952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350050150451354, + "loss": 2.944, + "theoretical_loss": 3.6130588058698665, + "tokens_seen": 1111359488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349949849548646, + "loss": 2.7659, + "theoretical_loss": 3.613038924219321, + "tokens_seen": 1111425024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349849548645938, + "loss": 2.7844, + "theoretical_loss": 3.6130190440693095, + "tokens_seen": 1111490560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033497492477432296, + "loss": 2.8088, + "theoretical_loss": 3.6129991654196294, + "tokens_seen": 1111556096 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033496489468405215, + "loss": 2.7411, + "theoretical_loss": 3.61297928827008, + "tokens_seen": 1111621632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349548645937813, + "loss": 2.8393, + "theoretical_loss": 3.612959412620459, + "tokens_seen": 1111687168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033494483450351056, + "loss": 2.7167, + "theoretical_loss": 3.612939538470566, + "tokens_seen": 1111752704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349348044132397, + "loss": 2.8403, + "theoretical_loss": 3.612919665820198, + "tokens_seen": 1111818240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349247743229689, + "loss": 2.7531, + "theoretical_loss": 3.612899794669154, + "tokens_seen": 1111883776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033491474423269805, + "loss": 2.5888, + "theoretical_loss": 3.6128799250172325, + "tokens_seen": 1111949312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349047141424273, + "loss": 2.6246, + "theoretical_loss": 3.6128600568642324, + "tokens_seen": 1112014848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033489468405215647, + "loss": 2.6785, + "theoretical_loss": 3.612840190209952, + "tokens_seen": 1112080384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033488465396188565, + "loss": 2.512, + "theoretical_loss": 3.61282032505419, + "tokens_seen": 1112145920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033487462387161483, + "loss": 2.6301, + "theoretical_loss": 3.612800461396745, + "tokens_seen": 1112211456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033486459378134406, + "loss": 2.7051, + "theoretical_loss": 3.612780599237416, + "tokens_seen": 1112276992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348545636910732, + "loss": 2.7752, + "theoretical_loss": 3.612760738576001, + "tokens_seen": 1112342528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348445336008024, + "loss": 2.7236, + "theoretical_loss": 3.6127408794123, + "tokens_seen": 1112408064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 617863, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2659218311309814, + "objective/train/theoretical_loss": 3.6127210217461108, + "objective/train/tokens_used": 1132933600, + "theoretical_loss": 3.6127210217461108, + "tokens_seen": 1112473600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348345035105316, + "loss": 2.6578, + "theoretical_loss": 3.6127210217461108, + "tokens_seen": 1112473600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348244734202608, + "loss": 3.0378, + "theoretical_loss": 3.6127011655772323, + "tokens_seen": 1112539136 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033481444332999, + "loss": 3.0138, + "theoretical_loss": 3.6126813109054643, + "tokens_seen": 1112604672 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033480441323971915, + "loss": 2.7494, + "theoretical_loss": 3.6126614577306047, + "tokens_seen": 1112670208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347943831494484, + "loss": 2.7871, + "theoretical_loss": 3.6126416060524535, + "tokens_seen": 1112735744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347843530591775, + "loss": 2.7748, + "theoretical_loss": 3.6126217558708085, + "tokens_seen": 1112801280 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033477432296890675, + "loss": 2.8669, + "theoretical_loss": 3.6126019071854696, + "tokens_seen": 1112866816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033476429287863593, + "loss": 2.4874, + "theoretical_loss": 3.612582059996236, + "tokens_seen": 1112932352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347542627883651, + "loss": 2.7502, + "theoretical_loss": 3.6125622143029066, + "tokens_seen": 1112997888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347442326980943, + "loss": 2.6639, + "theoretical_loss": 3.6125423701052806, + "tokens_seen": 1113063424 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033473420260782353, + "loss": 2.7515, + "theoretical_loss": 3.612522527403157, + "tokens_seen": 1113128960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033472417251755265, + "loss": 2.6853, + "theoretical_loss": 3.6125026861963354, + "tokens_seen": 1113194496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347141424272819, + "loss": 2.7526, + "theoretical_loss": 3.612482846484615, + "tokens_seen": 1113260032 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334704112337011, + "loss": 2.6622, + "theoretical_loss": 3.6124630082677944, + "tokens_seen": 1113325568 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033469408224674025, + "loss": 2.6131, + "theoretical_loss": 3.6124431715456744, + "tokens_seen": 1113391104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033468405215646943, + "loss": 2.6075, + "theoretical_loss": 3.612423336318053, + "tokens_seen": 1113456640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346740220661986, + "loss": 2.7197, + "theoretical_loss": 3.6124035025847308, + "tokens_seen": 1113522176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346639919759278, + "loss": 2.5603, + "theoretical_loss": 3.6123836703455066, + "tokens_seen": 1113587712 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334653961885657, + "loss": 3.03, + "theoretical_loss": 3.6123638396001807, + "tokens_seen": 1113653248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033464393179538616, + "loss": 2.8079, + "theoretical_loss": 3.6123440103485516, + "tokens_seen": 1113718784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346339017051154, + "loss": 2.8281, + "theoretical_loss": 3.6123241825904193, + "tokens_seen": 1113784320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346238716148445, + "loss": 2.4955, + "theoretical_loss": 3.6123043563255837, + "tokens_seen": 1113849856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033461384152457376, + "loss": 2.8836, + "theoretical_loss": 3.6122845315538443, + "tokens_seen": 1113915392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346038114343029, + "loss": 2.9342, + "theoretical_loss": 3.6122647082750006, + "tokens_seen": 1113980928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345937813440321, + "loss": 2.7755, + "theoretical_loss": 3.6122448864888526, + "tokens_seen": 1114046464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 618528, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8394312858581543, + "objective/train/theoretical_loss": 3.612225066195201, + "objective/train/tokens_used": 1134572000, + "theoretical_loss": 3.612225066195201, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345837512537613, + "loss": 2.8486, + "theoretical_loss": 3.612225066195201, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345737211634905, + "loss": 2.487, + "theoretical_loss": 3.6122052473938435, + "tokens_seen": 1114177536 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033456369107321966, + "loss": 2.8196, + "theoretical_loss": 3.612185430084582, + "tokens_seen": 1114243072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345536609829489, + "loss": 2.7459, + "theoretical_loss": 3.6121656142672154, + "tokens_seen": 1114308608 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334543630892678, + "loss": 2.6842, + "theoretical_loss": 3.612145799941544, + "tokens_seen": 1114374144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033453360080240726, + "loss": 2.6307, + "theoretical_loss": 3.612125987107367, + "tokens_seen": 1114439680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345235707121364, + "loss": 2.8632, + "theoretical_loss": 3.6121061757644863, + "tokens_seen": 1114505216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345135406218656, + "loss": 2.64, + "theoretical_loss": 3.6120863659127, + "tokens_seen": 1114570752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345035105315948, + "loss": 2.6673, + "theoretical_loss": 3.612066557551809, + "tokens_seen": 1114636288 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334493480441324, + "loss": 2.9368, + "theoretical_loss": 3.6120467506816136, + "tokens_seen": 1114701824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033448345035105316, + "loss": 2.9501, + "theoretical_loss": 3.612026945301914, + "tokens_seen": 1114767360 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033447342026078235, + "loss": 2.9252, + "theoretical_loss": 3.6120071414125103, + "tokens_seen": 1114832896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344633901705115, + "loss": 2.6258, + "theoretical_loss": 3.611987339013202, + "tokens_seen": 1114898432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033445336008024076, + "loss": 2.807, + "theoretical_loss": 3.611967538103791, + "tokens_seen": 1114963968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344433299899699, + "loss": 2.8856, + "theoretical_loss": 3.611947738684076, + "tokens_seen": 1115029504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344332998996991, + "loss": 2.7013, + "theoretical_loss": 3.6119279407538585, + "tokens_seen": 1115095040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033442326980942825, + "loss": 2.6865, + "theoretical_loss": 3.611908144312938, + "tokens_seen": 1115160576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344132397191575, + "loss": 2.9106, + "theoretical_loss": 3.6118883493611165, + "tokens_seen": 1115226112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033440320962888667, + "loss": 2.7511, + "theoretical_loss": 3.6118685558981927, + "tokens_seen": 1115291648 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033439317953861585, + "loss": 2.6814, + "theoretical_loss": 3.611848763923968, + "tokens_seen": 1115357184 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033438314944834503, + "loss": 2.902, + "theoretical_loss": 3.611828973438243, + "tokens_seen": 1115422720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033437311935807427, + "loss": 2.7772, + "theoretical_loss": 3.611809184440818, + "tokens_seen": 1115488256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343630892678034, + "loss": 2.8768, + "theoretical_loss": 3.6117893969314947, + "tokens_seen": 1115553792 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033435305917753263, + "loss": 2.3855, + "theoretical_loss": 3.6117696109100716, + "tokens_seen": 1115619328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033434302908726175, + "loss": 2.8104, + "theoretical_loss": 3.611749826376352, + "tokens_seen": 1115684864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 619820, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4851090908050537, + "objective/train/theoretical_loss": 3.6117300433301347, + "objective/train/tokens_used": 1136210400, + "theoretical_loss": 3.6117300433301347, + "tokens_seen": 1115750400 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334332998996991, + "loss": 2.7111, + "theoretical_loss": 3.6117300433301347, + "tokens_seen": 1115750400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033432296890672017, + "loss": 2.8355, + "theoretical_loss": 3.611710261771221, + "tokens_seen": 1115815936 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033431293881644935, + "loss": 2.7384, + "theoretical_loss": 3.611690481699412, + "tokens_seen": 1115881472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033430290872617853, + "loss": 2.6487, + "theoretical_loss": 3.611670703114509, + "tokens_seen": 1115947008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342928786359077, + "loss": 2.8993, + "theoretical_loss": 3.6116509260163125, + "tokens_seen": 1116012544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342828485456369, + "loss": 2.6758, + "theoretical_loss": 3.611631150404623, + "tokens_seen": 1116078080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033427281845536613, + "loss": 2.5941, + "theoretical_loss": 3.611611376279242, + "tokens_seen": 1116143616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033426278836509526, + "loss": 2.7803, + "theoretical_loss": 3.6115916036399702, + "tokens_seen": 1116209152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342527582748245, + "loss": 2.5758, + "theoretical_loss": 3.611571832486609, + "tokens_seen": 1116274688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342427281845536, + "loss": 2.6273, + "theoretical_loss": 3.6115520628189595, + "tokens_seen": 1116340224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033423269809428285, + "loss": 2.9422, + "theoretical_loss": 3.611532294636823, + "tokens_seen": 1116405760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033422266800401204, + "loss": 2.5773, + "theoretical_loss": 3.6115125279399996, + "tokens_seen": 1116471296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342126379137412, + "loss": 2.7014, + "theoretical_loss": 3.6114927627282922, + "tokens_seen": 1116536832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342026078234704, + "loss": 2.6084, + "theoretical_loss": 3.611472999001501, + "tokens_seen": 1116602368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033419257773319963, + "loss": 2.8009, + "theoretical_loss": 3.611453236759427, + "tokens_seen": 1116667904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033418254764292876, + "loss": 2.757, + "theoretical_loss": 3.6114334760018725, + "tokens_seen": 1116733440 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334172517552658, + "loss": 2.8419, + "theoretical_loss": 3.6114137167286384, + "tokens_seen": 1116798976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341624874623871, + "loss": 2.444, + "theoretical_loss": 3.6113939589395256, + "tokens_seen": 1116864512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033415245737211636, + "loss": 2.7549, + "theoretical_loss": 3.6113742026343365, + "tokens_seen": 1116930048 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033414242728184554, + "loss": 2.7793, + "theoretical_loss": 3.6113544478128725, + "tokens_seen": 1116995584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341323971915747, + "loss": 2.4836, + "theoretical_loss": 3.6113346944749343, + "tokens_seen": 1117061120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341223671013039, + "loss": 2.7046, + "theoretical_loss": 3.611314942620324, + "tokens_seen": 1117126656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341123370110331, + "loss": 2.5733, + "theoretical_loss": 3.611295192248843, + "tokens_seen": 1117192192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033410230692076226, + "loss": 2.7553, + "theoretical_loss": 3.6112754433602934, + "tokens_seen": 1117257728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340922768304915, + "loss": 2.6006, + "theoretical_loss": 3.6112556959544766, + "tokens_seen": 1117323264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 620252, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6974408626556396, + "objective/train/theoretical_loss": 3.611235950031194, + "objective/train/tokens_used": 1137848800, + "theoretical_loss": 3.611235950031194, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340822467402207, + "loss": 2.6483, + "theoretical_loss": 3.611235950031194, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033407221664994986, + "loss": 2.8296, + "theoretical_loss": 3.6112162055902477, + "tokens_seen": 1117454336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340621865596791, + "loss": 2.6342, + "theoretical_loss": 3.61119646263144, + "tokens_seen": 1117519872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340521564694082, + "loss": 2.7249, + "theoretical_loss": 3.6111767211545716, + "tokens_seen": 1117585408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033404212637913746, + "loss": 2.6615, + "theoretical_loss": 3.6111569811594455, + "tokens_seen": 1117650944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340320962888666, + "loss": 2.8549, + "theoretical_loss": 3.6111372426458628, + "tokens_seen": 1117716480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340220661985958, + "loss": 2.7704, + "theoretical_loss": 3.611117505613625, + "tokens_seen": 1117782016 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334012036108325, + "loss": 2.79, + "theoretical_loss": 3.6110977700625355, + "tokens_seen": 1117847552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340020060180542, + "loss": 2.5227, + "theoretical_loss": 3.6110780359923957, + "tokens_seen": 1117913088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033399197592778336, + "loss": 3.0267, + "theoretical_loss": 3.6110583034030075, + "tokens_seen": 1117978624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033398194583751255, + "loss": 2.4724, + "theoretical_loss": 3.6110385722941727, + "tokens_seen": 1118044160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339719157472417, + "loss": 2.6887, + "theoretical_loss": 3.611018842665694, + "tokens_seen": 1118109696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033396188565697096, + "loss": 2.7518, + "theoretical_loss": 3.6109991145173734, + "tokens_seen": 1118175232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339518555667001, + "loss": 2.5372, + "theoretical_loss": 3.6109793878490133, + "tokens_seen": 1118240768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339418254764293, + "loss": 2.7651, + "theoretical_loss": 3.6109596626604157, + "tokens_seen": 1118306304 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033393179538615845, + "loss": 2.7977, + "theoretical_loss": 3.6109399389513825, + "tokens_seen": 1118371840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339217652958877, + "loss": 2.728, + "theoretical_loss": 3.610920216721717, + "tokens_seen": 1118437376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033391173520561687, + "loss": 2.6523, + "theoretical_loss": 3.610900495971221, + "tokens_seen": 1118502912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033390170511534605, + "loss": 2.7489, + "theoretical_loss": 3.6108807766996964, + "tokens_seen": 1118568448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033389167502507523, + "loss": 2.8485, + "theoretical_loss": 3.6108610589069468, + "tokens_seen": 1118633984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033388164493480447, + "loss": 2.7418, + "theoretical_loss": 3.610841342592774, + "tokens_seen": 1118699520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338716148445336, + "loss": 2.6925, + "theoretical_loss": 3.6108216277569802, + "tokens_seen": 1118765056 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033386158475426283, + "loss": 2.8039, + "theoretical_loss": 3.610801914399369, + "tokens_seen": 1118830592 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033385155466399195, + "loss": 2.7368, + "theoretical_loss": 3.610782202519742, + "tokens_seen": 1118896128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338415245737212, + "loss": 2.8413, + "theoretical_loss": 3.6107624921179013, + "tokens_seen": 1118961664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 621194, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.00057315826416, + "objective/train/theoretical_loss": 3.6107427831936514, + "objective/train/tokens_used": 1139487200, + "theoretical_loss": 3.6107427831936514, + "tokens_seen": 1119027200 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033383149448345037, + "loss": 2.9902, + "theoretical_loss": 3.6107427831936514, + "tokens_seen": 1119027200 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033382146439317955, + "loss": 2.743, + "theoretical_loss": 3.610723075746794, + "tokens_seen": 1119092736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033381143430290873, + "loss": 2.6116, + "theoretical_loss": 3.6107033697771316, + "tokens_seen": 1119158272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338014042126379, + "loss": 3.0118, + "theoretical_loss": 3.6106836652844674, + "tokens_seen": 1119223808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337913741223671, + "loss": 2.5351, + "theoretical_loss": 3.610663962268604, + "tokens_seen": 1119289344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033378134403209633, + "loss": 2.6608, + "theoretical_loss": 3.6106442607293445, + "tokens_seen": 1119354880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033377131394182546, + "loss": 2.6882, + "theoretical_loss": 3.6106245606664915, + "tokens_seen": 1119420416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337612838515547, + "loss": 2.6599, + "theoretical_loss": 3.610604862079848, + "tokens_seen": 1119485952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337512537612838, + "loss": 2.7627, + "theoretical_loss": 3.6105851649692173, + "tokens_seen": 1119551488 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033374122367101306, + "loss": 2.6458, + "theoretical_loss": 3.610565469334402, + "tokens_seen": 1119617024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033373119358074224, + "loss": 2.6959, + "theoretical_loss": 3.610545775175205, + "tokens_seen": 1119682560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337211634904714, + "loss": 2.746, + "theoretical_loss": 3.61052608249143, + "tokens_seen": 1119748096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337111334002006, + "loss": 2.6225, + "theoretical_loss": 3.61050639128288, + "tokens_seen": 1119813632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033370110330992983, + "loss": 2.785, + "theoretical_loss": 3.610486701549357, + "tokens_seen": 1119879168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033369107321965896, + "loss": 2.8137, + "theoretical_loss": 3.610467013290666, + "tokens_seen": 1119944704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336810431293882, + "loss": 2.9026, + "theoretical_loss": 3.61044732650661, + "tokens_seen": 1120010240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336710130391173, + "loss": 2.7828, + "theoretical_loss": 3.6104276411969907, + "tokens_seen": 1120075776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033366098294884656, + "loss": 2.7948, + "theoretical_loss": 3.6104079573616126, + "tokens_seen": 1120141312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033365095285857574, + "loss": 2.748, + "theoretical_loss": 3.610388275000279, + "tokens_seen": 1120206848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336409227683049, + "loss": 2.8541, + "theoretical_loss": 3.610368594112793, + "tokens_seen": 1120272384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336308926780341, + "loss": 2.7131, + "theoretical_loss": 3.6103489146989585, + "tokens_seen": 1120337920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336208625877633, + "loss": 2.4622, + "theoretical_loss": 3.610329236758578, + "tokens_seen": 1120403456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033361083249749246, + "loss": 2.7207, + "theoretical_loss": 3.6103095602914563, + "tokens_seen": 1120468992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336008024072217, + "loss": 2.6971, + "theoretical_loss": 3.610289885297396, + "tokens_seen": 1120534528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335907723169508, + "loss": 3.0744, + "theoretical_loss": 3.610270211776201, + "tokens_seen": 1120600064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 621844, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9886622428894043, + "objective/train/theoretical_loss": 3.6102505397276743, + "objective/train/tokens_used": 1141125600, + "theoretical_loss": 3.6102505397276743, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033358074222668006, + "loss": 2.812, + "theoretical_loss": 3.6102505397276743, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033357071213640924, + "loss": 2.6918, + "theoretical_loss": 3.6102308691516205, + "tokens_seen": 1120731136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335606820461384, + "loss": 2.6555, + "theoretical_loss": 3.610211200047843, + "tokens_seen": 1120796672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335506519558676, + "loss": 2.9233, + "theoretical_loss": 3.6101915324161453, + "tokens_seen": 1120862208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335406218655968, + "loss": 2.6204, + "theoretical_loss": 3.6101718662563314, + "tokens_seen": 1120927744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033353059177532597, + "loss": 2.5356, + "theoretical_loss": 3.610152201568205, + "tokens_seen": 1120993280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335205616850552, + "loss": 2.8194, + "theoretical_loss": 3.6101325383515697, + "tokens_seen": 1121058816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033351053159478433, + "loss": 2.8566, + "theoretical_loss": 3.6101128766062294, + "tokens_seen": 1121124352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033350050150451356, + "loss": 2.7643, + "theoretical_loss": 3.6100932163319888, + "tokens_seen": 1121189888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334904714142427, + "loss": 2.7626, + "theoretical_loss": 3.6100735575286507, + "tokens_seen": 1121255424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334804413239719, + "loss": 2.5128, + "theoretical_loss": 3.6100539001960197, + "tokens_seen": 1121320960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334704112337011, + "loss": 2.7467, + "theoretical_loss": 3.6100342443339004, + "tokens_seen": 1121386496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334603811434303, + "loss": 2.7479, + "theoretical_loss": 3.6100145899420957, + "tokens_seen": 1121452032 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033345035105315947, + "loss": 2.8238, + "theoretical_loss": 3.6099949370204105, + "tokens_seen": 1121517568 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033344032096288865, + "loss": 2.6709, + "theoretical_loss": 3.6099752855686487, + "tokens_seen": 1121583104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033343029087261783, + "loss": 2.7122, + "theoretical_loss": 3.6099556355866143, + "tokens_seen": 1121648640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033342026078234707, + "loss": 2.7043, + "theoretical_loss": 3.6099359870741115, + "tokens_seen": 1121714176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334102306920762, + "loss": 2.6631, + "theoretical_loss": 3.609916340030945, + "tokens_seen": 1121779712 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033340020060180543, + "loss": 2.5839, + "theoretical_loss": 3.609896694456919, + "tokens_seen": 1121845248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333901705115346, + "loss": 2.9894, + "theoretical_loss": 3.6098770503518374, + "tokens_seen": 1121910784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333801404212638, + "loss": 2.6817, + "theoretical_loss": 3.609857407715505, + "tokens_seen": 1121976320 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333370110330993, + "loss": 2.2964, + "theoretical_loss": 3.609837766547726, + "tokens_seen": 1122041856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033336008024072215, + "loss": 2.6476, + "theoretical_loss": 3.609818126848305, + "tokens_seen": 1122107392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033335005015045134, + "loss": 2.7987, + "theoretical_loss": 3.609798488617046, + "tokens_seen": 1122172928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033334002006018057, + "loss": 2.6787, + "theoretical_loss": 3.609778851853754, + "tokens_seen": 1122238464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 622614, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.409770965576172, + "objective/train/theoretical_loss": 3.609759216558233, + "objective/train/tokens_used": 1142764000, + "theoretical_loss": 3.609759216558233, + "tokens_seen": 1122304000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033332998996990975, + "loss": 2.6987, + "theoretical_loss": 3.609759216558233, + "tokens_seen": 1122304000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033331995987963893, + "loss": 2.4774, + "theoretical_loss": 3.6097395827302887, + "tokens_seen": 1122369536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333099297893681, + "loss": 2.7174, + "theoretical_loss": 3.609719950369725, + "tokens_seen": 1122435072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332998996990973, + "loss": 2.8698, + "theoretical_loss": 3.609700319476346, + "tokens_seen": 1122500608 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033328986960882653, + "loss": 2.7169, + "theoretical_loss": 3.6096806900499576, + "tokens_seen": 1122566144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033327983951855566, + "loss": 2.6388, + "theoretical_loss": 3.6096610620903635, + "tokens_seen": 1122631680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332698094282849, + "loss": 2.608, + "theoretical_loss": 3.609641435597369, + "tokens_seen": 1122697216 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333259779338014, + "loss": 2.9848, + "theoretical_loss": 3.609621810570779, + "tokens_seen": 1122762752 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033324974924774326, + "loss": 2.5629, + "theoretical_loss": 3.609602187010398, + "tokens_seen": 1122828288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033323971915747244, + "loss": 2.6, + "theoretical_loss": 3.6095825649160314, + "tokens_seen": 1122893824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332296890672016, + "loss": 2.7688, + "theoretical_loss": 3.6095629442874833, + "tokens_seen": 1122959360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332196589769308, + "loss": 2.555, + "theoretical_loss": 3.6095433251245597, + "tokens_seen": 1123024896 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033320962888666003, + "loss": 2.859, + "theoretical_loss": 3.6095237074270647, + "tokens_seen": 1123090432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033319959879638916, + "loss": 2.8304, + "theoretical_loss": 3.609504091194804, + "tokens_seen": 1123155968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331895687061184, + "loss": 2.5043, + "theoretical_loss": 3.609484476427582, + "tokens_seen": 1123221504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331795386158475, + "loss": 2.7801, + "theoretical_loss": 3.609464863125204, + "tokens_seen": 1123287040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033316950852557676, + "loss": 2.6406, + "theoretical_loss": 3.609445251287476, + "tokens_seen": 1123352576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033315947843530594, + "loss": 2.6735, + "theoretical_loss": 3.609425640914202, + "tokens_seen": 1123418112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331494483450351, + "loss": 2.6454, + "theoretical_loss": 3.609406032005188, + "tokens_seen": 1123483648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331394182547643, + "loss": 2.6867, + "theoretical_loss": 3.609386424560239, + "tokens_seen": 1123549184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331293881644935, + "loss": 2.5992, + "theoretical_loss": 3.6093668185791605, + "tokens_seen": 1123614720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033311935807422266, + "loss": 2.6658, + "theoretical_loss": 3.6093472140617573, + "tokens_seen": 1123680256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331093279839519, + "loss": 2.7022, + "theoretical_loss": 3.609327611007835, + "tokens_seen": 1123745792 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333099297893681, + "loss": 2.7329, + "theoretical_loss": 3.6093080094172, + "tokens_seen": 1123811328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033308926780341026, + "loss": 2.5589, + "theoretical_loss": 3.6092884092896558, + "tokens_seen": 1123876864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 624048, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0764658451080322, + "objective/train/theoretical_loss": 3.60926881062501, + "objective/train/tokens_used": 1144402400, + "theoretical_loss": 3.60926881062501, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033307923771313944, + "loss": 2.8697, + "theoretical_loss": 3.60926881062501, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330692076228686, + "loss": 2.7104, + "theoretical_loss": 3.609249213423066, + "tokens_seen": 1124007936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330591775325978, + "loss": 2.8209, + "theoretical_loss": 3.6092296176836314, + "tokens_seen": 1124073472 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333049147442327, + "loss": 2.7686, + "theoretical_loss": 3.6092100234065105, + "tokens_seen": 1124139008 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033303911735205617, + "loss": 2.6625, + "theoretical_loss": 3.609190430591509, + "tokens_seen": 1124204544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330290872617854, + "loss": 2.7168, + "theoretical_loss": 3.609170839238433, + "tokens_seen": 1124270080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033301905717151453, + "loss": 2.9399, + "theoretical_loss": 3.6091512493470885, + "tokens_seen": 1124335616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033300902708124376, + "loss": 2.7011, + "theoretical_loss": 3.60913166091728, + "tokens_seen": 1124401152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329989969909729, + "loss": 2.6836, + "theoretical_loss": 3.6091120739488147, + "tokens_seen": 1124466688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329889669007021, + "loss": 2.8206, + "theoretical_loss": 3.609092488441498, + "tokens_seen": 1124532224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329789368104313, + "loss": 2.6126, + "theoretical_loss": 3.609072904395135, + "tokens_seen": 1124597760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329689067201605, + "loss": 2.6087, + "theoretical_loss": 3.609053321809532, + "tokens_seen": 1124663296 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033295887662988967, + "loss": 2.7225, + "theoretical_loss": 3.609033740684496, + "tokens_seen": 1124728832 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033294884653961885, + "loss": 2.9933, + "theoretical_loss": 3.6090141610198314, + "tokens_seen": 1124794368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033293881644934803, + "loss": 2.831, + "theoretical_loss": 3.6089945828153454, + "tokens_seen": 1124859904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033292878635907727, + "loss": 2.7406, + "theoretical_loss": 3.608975006070843, + "tokens_seen": 1124925440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329187562688064, + "loss": 2.6774, + "theoretical_loss": 3.608955430786131, + "tokens_seen": 1124990976 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033290872617853563, + "loss": 2.7052, + "theoretical_loss": 3.6089358569610153, + "tokens_seen": 1125056512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328986960882648, + "loss": 2.6838, + "theoretical_loss": 3.608916284595302, + "tokens_seen": 1125122048 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332888665997994, + "loss": 2.7067, + "theoretical_loss": 3.6088967136887975, + "tokens_seen": 1125187584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328786359077232, + "loss": 2.7984, + "theoretical_loss": 3.608877144241308, + "tokens_seen": 1125253120 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033286860581745235, + "loss": 2.9188, + "theoretical_loss": 3.6088575762526394, + "tokens_seen": 1125318656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033285857572718154, + "loss": 2.7095, + "theoretical_loss": 3.6088380097225983, + "tokens_seen": 1125384192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033284854563691077, + "loss": 2.734, + "theoretical_loss": 3.608818444650991, + "tokens_seen": 1125449728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328385155466399, + "loss": 2.5812, + "theoretical_loss": 3.6087988810376235, + "tokens_seen": 1125515264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 624451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6627368927001953, + "objective/train/theoretical_loss": 3.608779318882303, + "objective/train/tokens_used": 1146040800, + "theoretical_loss": 3.608779318882303, + "tokens_seen": 1125580800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033282848545636913, + "loss": 2.6308, + "theoretical_loss": 3.608779318882303, + "tokens_seen": 1125580800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033281845536609826, + "loss": 2.5499, + "theoretical_loss": 3.608759758184836, + "tokens_seen": 1125646336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328084252758275, + "loss": 2.8517, + "theoretical_loss": 3.6087401989450276, + "tokens_seen": 1125711872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327983951855567, + "loss": 2.6624, + "theoretical_loss": 3.6087206411626855, + "tokens_seen": 1125777408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033278836509528586, + "loss": 2.751, + "theoretical_loss": 3.6087010848376164, + "tokens_seen": 1125842944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033277833500501504, + "loss": 2.5501, + "theoretical_loss": 3.608681529969626, + "tokens_seen": 1125908480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327683049147442, + "loss": 2.6393, + "theoretical_loss": 3.6086619765585217, + "tokens_seen": 1125974016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327582748244734, + "loss": 2.6149, + "theoretical_loss": 3.6086424246041098, + "tokens_seen": 1126039552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033274824473420264, + "loss": 2.805, + "theoretical_loss": 3.6086228741061976, + "tokens_seen": 1126105088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033273821464393176, + "loss": 2.7527, + "theoretical_loss": 3.608603325064591, + "tokens_seen": 1126170624 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332728184553661, + "loss": 2.9454, + "theoretical_loss": 3.608583777479097, + "tokens_seen": 1126236160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327181544633902, + "loss": 2.8227, + "theoretical_loss": 3.6085642313495225, + "tokens_seen": 1126301696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033270812437311936, + "loss": 2.7314, + "theoretical_loss": 3.6085446866756747, + "tokens_seen": 1126367232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033269809428284854, + "loss": 2.6964, + "theoretical_loss": 3.60852514345736, + "tokens_seen": 1126432768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326880641925777, + "loss": 2.8856, + "theoretical_loss": 3.6085056016943855, + "tokens_seen": 1126498304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326780341023069, + "loss": 2.9844, + "theoretical_loss": 3.608486061386558, + "tokens_seen": 1126563840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033266800401203614, + "loss": 2.8392, + "theoretical_loss": 3.6084665225336856, + "tokens_seen": 1126629376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033265797392176527, + "loss": 2.8072, + "theoretical_loss": 3.608446985135574, + "tokens_seen": 1126694912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326479438314945, + "loss": 2.9037, + "theoretical_loss": 3.6084274491920305, + "tokens_seen": 1126760448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033263791374122363, + "loss": 2.713, + "theoretical_loss": 3.6084079147028625, + "tokens_seen": 1126825984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033262788365095286, + "loss": 2.622, + "theoretical_loss": 3.608388381667877, + "tokens_seen": 1126891520 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033261785356068205, + "loss": 2.7095, + "theoretical_loss": 3.6083688500868814, + "tokens_seen": 1126957056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326078234704112, + "loss": 2.6724, + "theoretical_loss": 3.6083493199596828, + "tokens_seen": 1127022592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325977933801404, + "loss": 2.6232, + "theoretical_loss": 3.6083297912860886, + "tokens_seen": 1127088128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033258776328986964, + "loss": 2.7234, + "theoretical_loss": 3.6083102640659055, + "tokens_seen": 1127153664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 625621, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.037780284881592, + "objective/train/theoretical_loss": 3.608290738298942, + "objective/train/tokens_used": 1147679200, + "theoretical_loss": 3.608290738298942, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325777331995988, + "loss": 2.6553, + "theoretical_loss": 3.608290738298942, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332567703109328, + "loss": 2.813, + "theoretical_loss": 3.608271213985004, + "tokens_seen": 1127284736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325576730190572, + "loss": 2.5845, + "theoretical_loss": 3.6082516911239004, + "tokens_seen": 1127350272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033254764292878637, + "loss": 2.8016, + "theoretical_loss": 3.608232169715438, + "tokens_seen": 1127415808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325376128385156, + "loss": 2.8117, + "theoretical_loss": 3.6082126497594236, + "tokens_seen": 1127481344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033252758274824473, + "loss": 2.7862, + "theoretical_loss": 3.6081931312556654, + "tokens_seen": 1127546880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033251755265797396, + "loss": 2.8977, + "theoretical_loss": 3.608173614203971, + "tokens_seen": 1127612416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325075225677031, + "loss": 2.6172, + "theoretical_loss": 3.608154098604148, + "tokens_seen": 1127677952 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033249749247743233, + "loss": 2.8162, + "theoretical_loss": 3.6081345844560033, + "tokens_seen": 1127743488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324874623871615, + "loss": 2.5889, + "theoretical_loss": 3.6081150717593458, + "tokens_seen": 1127809024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324774322968907, + "loss": 2.8404, + "theoretical_loss": 3.6080955605139824, + "tokens_seen": 1127874560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033246740220661987, + "loss": 2.7708, + "theoretical_loss": 3.6080760507197214, + "tokens_seen": 1127940096 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033245737211634905, + "loss": 2.7785, + "theoretical_loss": 3.6080565423763695, + "tokens_seen": 1128005632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033244734202607823, + "loss": 2.9172, + "theoretical_loss": 3.6080370354837354, + "tokens_seen": 1128071168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033243731193580747, + "loss": 2.5575, + "theoretical_loss": 3.608017530041627, + "tokens_seen": 1128136704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324272818455366, + "loss": 2.6802, + "theoretical_loss": 3.6079980260498514, + "tokens_seen": 1128202240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033241725175526583, + "loss": 2.6072, + "theoretical_loss": 3.6079785235082174, + "tokens_seen": 1128267776 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332407221664995, + "loss": 2.6779, + "theoretical_loss": 3.6079590224165328, + "tokens_seen": 1128333312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323971915747242, + "loss": 2.6611, + "theoretical_loss": 3.607939522774605, + "tokens_seen": 1128398848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323871614844534, + "loss": 2.7856, + "theoretical_loss": 3.607920024582242, + "tokens_seen": 1128464384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033237713139418255, + "loss": 2.6113, + "theoretical_loss": 3.607900527839253, + "tokens_seen": 1128529920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033236710130391174, + "loss": 2.8294, + "theoretical_loss": 3.6078810325454453, + "tokens_seen": 1128595456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033235707121364097, + "loss": 2.7486, + "theoretical_loss": 3.607861538700627, + "tokens_seen": 1128660992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323470411233701, + "loss": 2.9031, + "theoretical_loss": 3.607842046304606, + "tokens_seen": 1128726528 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033233701103309933, + "loss": 2.8622, + "theoretical_loss": 3.607822555357192, + "tokens_seen": 1128792064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 626861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9778614044189453, + "objective/train/theoretical_loss": 3.6078030658581914, + "objective/train/tokens_used": 1149317600, + "theoretical_loss": 3.6078030658581914, + "tokens_seen": 1128857600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033232698094282846, + "loss": 2.7505, + "theoretical_loss": 3.6078030658581914, + "tokens_seen": 1128857600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323169508525577, + "loss": 2.6772, + "theoretical_loss": 3.607783577807413, + "tokens_seen": 1128923136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323069207622869, + "loss": 2.654, + "theoretical_loss": 3.6077640912046665, + "tokens_seen": 1128988672 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033229689067201606, + "loss": 2.6944, + "theoretical_loss": 3.607744606049758, + "tokens_seen": 1129054208 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033228686058174524, + "loss": 2.7522, + "theoretical_loss": 3.6077251223424973, + "tokens_seen": 1129119744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322768304914744, + "loss": 2.7485, + "theoretical_loss": 3.607705640082693, + "tokens_seen": 1129185280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322668004012036, + "loss": 2.78, + "theoretical_loss": 3.607686159270153, + "tokens_seen": 1129250816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033225677031093284, + "loss": 2.8463, + "theoretical_loss": 3.6076666799046855, + "tokens_seen": 1129316352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033224674022066196, + "loss": 2.7895, + "theoretical_loss": 3.6076472019861, + "tokens_seen": 1129381888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322367101303912, + "loss": 2.5306, + "theoretical_loss": 3.607627725514204, + "tokens_seen": 1129447424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322266800401204, + "loss": 2.5914, + "theoretical_loss": 3.6076082504888074, + "tokens_seen": 1129512960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033221664994984956, + "loss": 2.6608, + "theoretical_loss": 3.6075887769097177, + "tokens_seen": 1129578496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033220661985957874, + "loss": 2.8178, + "theoretical_loss": 3.607569304776744, + "tokens_seen": 1129644032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321965897693079, + "loss": 2.6823, + "theoretical_loss": 3.607549834089695, + "tokens_seen": 1129709568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321865596790371, + "loss": 2.7135, + "theoretical_loss": 3.6075303648483796, + "tokens_seen": 1129775104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033217652958876634, + "loss": 2.7891, + "theoretical_loss": 3.607510897052607, + "tokens_seen": 1129840640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033216649949849547, + "loss": 2.5299, + "theoretical_loss": 3.6074914307021846, + "tokens_seen": 1129906176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321564694082247, + "loss": 2.7886, + "theoretical_loss": 3.6074719657969228, + "tokens_seen": 1129971712 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033214643931795383, + "loss": 2.8238, + "theoretical_loss": 3.60745250233663, + "tokens_seen": 1130037248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033213640922768306, + "loss": 2.7838, + "theoretical_loss": 3.607433040321115, + "tokens_seen": 1130102784 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033212637913741225, + "loss": 2.7236, + "theoretical_loss": 3.6074135797501867, + "tokens_seen": 1130168320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321163490471414, + "loss": 2.9937, + "theoretical_loss": 3.6073941206236544, + "tokens_seen": 1130233856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321063189568706, + "loss": 2.8337, + "theoretical_loss": 3.607374662941327, + "tokens_seen": 1130299392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033209628886659984, + "loss": 2.7751, + "theoretical_loss": 3.607355206703013, + "tokens_seen": 1130364928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033208625877632897, + "loss": 2.7562, + "theoretical_loss": 3.607335751908522, + "tokens_seen": 1130430464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 627282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7507452964782715, + "objective/train/theoretical_loss": 3.6073162985576643, + "objective/train/tokens_used": 1150956000, + "theoretical_loss": 3.6073162985576643, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320762286860582, + "loss": 2.7378, + "theoretical_loss": 3.6073162985576643, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033206619859578733, + "loss": 2.6933, + "theoretical_loss": 3.6072968466502475, + "tokens_seen": 1130561536 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033205616850551657, + "loss": 2.6401, + "theoretical_loss": 3.6072773961860816, + "tokens_seen": 1130627072 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033204613841524575, + "loss": 2.7799, + "theoretical_loss": 3.6072579471649755, + "tokens_seen": 1130692608 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033203610832497493, + "loss": 2.6876, + "theoretical_loss": 3.607238499586739, + "tokens_seen": 1130758144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320260782347041, + "loss": 2.7543, + "theoretical_loss": 3.6072190534511805, + "tokens_seen": 1130823680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320160481444333, + "loss": 2.8909, + "theoretical_loss": 3.60719960875811, + "tokens_seen": 1130889216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320060180541625, + "loss": 2.6733, + "theoretical_loss": 3.6071801655073377, + "tokens_seen": 1130954752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319959879638917, + "loss": 2.5583, + "theoretical_loss": 3.6071607236986716, + "tokens_seen": 1131020288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033198595787362084, + "loss": 2.9844, + "theoretical_loss": 3.607141283331922, + "tokens_seen": 1131085824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033197592778335007, + "loss": 2.933, + "theoretical_loss": 3.6071218444068984, + "tokens_seen": 1131151360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319658976930792, + "loss": 2.4985, + "theoretical_loss": 3.60710240692341, + "tokens_seen": 1131216896 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033195586760280843, + "loss": 2.7601, + "theoretical_loss": 3.607082970881267, + "tokens_seen": 1131282432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319458375125376, + "loss": 2.9582, + "theoretical_loss": 3.607063536280278, + "tokens_seen": 1131347968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319358074222668, + "loss": 2.6208, + "theoretical_loss": 3.607044103120254, + "tokens_seen": 1131413504 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331925777331996, + "loss": 2.9075, + "theoretical_loss": 3.6070246714010037, + "tokens_seen": 1131479040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319157472417252, + "loss": 2.8108, + "theoretical_loss": 3.6070052411223372, + "tokens_seen": 1131544576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033190571715145434, + "loss": 2.81, + "theoretical_loss": 3.6069858122840643, + "tokens_seen": 1131610112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318956870611836, + "loss": 2.696, + "theoretical_loss": 3.606966384885995, + "tokens_seen": 1131675648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318856569709127, + "loss": 2.6364, + "theoretical_loss": 3.6069469589279386, + "tokens_seen": 1131741184 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033187562688064194, + "loss": 2.7666, + "theoretical_loss": 3.6069275344097047, + "tokens_seen": 1131806720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318655967903711, + "loss": 2.8148, + "theoretical_loss": 3.606908111331105, + "tokens_seen": 1131872256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318555667001003, + "loss": 2.9294, + "theoretical_loss": 3.6068886896919476, + "tokens_seen": 1131937792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318455366098295, + "loss": 2.6665, + "theoretical_loss": 3.6068692694920426, + "tokens_seen": 1132003328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033183550651955866, + "loss": 2.5168, + "theoretical_loss": 3.6068498507312015, + "tokens_seen": 1132068864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 628386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4533891677856445, + "objective/train/theoretical_loss": 3.6068304334092334, + "objective/train/tokens_used": 1152594400, + "theoretical_loss": 3.6068304334092334, + "tokens_seen": 1132134400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318254764292879, + "loss": 2.6008, + "theoretical_loss": 3.6068304334092334, + "tokens_seen": 1132134400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318154463390171, + "loss": 2.8305, + "theoretical_loss": 3.606811017525948, + "tokens_seen": 1132199936 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033180541624874626, + "loss": 2.7182, + "theoretical_loss": 3.606791603081156, + "tokens_seen": 1132265472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033179538615847544, + "loss": 2.8093, + "theoretical_loss": 3.6067721900746674, + "tokens_seen": 1132331008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317853560682046, + "loss": 2.5651, + "theoretical_loss": 3.6067527785062925, + "tokens_seen": 1132396544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317753259779338, + "loss": 2.8494, + "theoretical_loss": 3.606733368375842, + "tokens_seen": 1132462080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033176529588766304, + "loss": 2.7811, + "theoretical_loss": 3.606713959683125, + "tokens_seen": 1132527616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033175526579739216, + "loss": 2.9312, + "theoretical_loss": 3.6066945524279532, + "tokens_seen": 1132593152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317452357071214, + "loss": 2.7798, + "theoretical_loss": 3.6066751466101357, + "tokens_seen": 1132658688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317352056168506, + "loss": 2.7387, + "theoretical_loss": 3.606655742229483, + "tokens_seen": 1132724224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033172517552657976, + "loss": 2.6532, + "theoretical_loss": 3.606636339285807, + "tokens_seen": 1132789760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033171514543630894, + "loss": 2.7307, + "theoretical_loss": 3.6066169377789166, + "tokens_seen": 1132855296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317051153460381, + "loss": 2.7641, + "theoretical_loss": 3.6065975377086232, + "tokens_seen": 1132920832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316950852557673, + "loss": 2.7869, + "theoretical_loss": 3.6065781390747365, + "tokens_seen": 1132986368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033168505516549654, + "loss": 2.6226, + "theoretical_loss": 3.606558741877068, + "tokens_seen": 1133051904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033167502507522567, + "loss": 2.7482, + "theoretical_loss": 3.6065393461154276, + "tokens_seen": 1133117440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316649949849549, + "loss": 2.4595, + "theoretical_loss": 3.6065199517896263, + "tokens_seen": 1133182976 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033165496489468403, + "loss": 2.7327, + "theoretical_loss": 3.6065005588994747, + "tokens_seen": 1133248512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033164493480441326, + "loss": 2.7135, + "theoretical_loss": 3.606481167444783, + "tokens_seen": 1133314048 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033163490471414245, + "loss": 2.6186, + "theoretical_loss": 3.6064617774253627, + "tokens_seen": 1133379584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316248746238716, + "loss": 2.9134, + "theoretical_loss": 3.606442388841024, + "tokens_seen": 1133445120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316148445336008, + "loss": 2.8247, + "theoretical_loss": 3.606423001691579, + "tokens_seen": 1133510656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033160481444333004, + "loss": 2.6597, + "theoretical_loss": 3.606403615976837, + "tokens_seen": 1133576192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033159478435305917, + "loss": 2.4916, + "theoretical_loss": 3.606384231696609, + "tokens_seen": 1133641728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315847542627884, + "loss": 2.9193, + "theoretical_loss": 3.6063648488507067, + "tokens_seen": 1133707264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 629006, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7128870487213135, + "objective/train/theoretical_loss": 3.606345467438941, + "objective/train/tokens_used": 1154232800, + "theoretical_loss": 3.606345467438941, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033157472417251753, + "loss": 2.7728, + "theoretical_loss": 3.606345467438941, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033156469408224677, + "loss": 2.7129, + "theoretical_loss": 3.6063260874611225, + "tokens_seen": 1133838336 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033155466399197595, + "loss": 2.7891, + "theoretical_loss": 3.6063067089170624, + "tokens_seen": 1133903872 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033154463390170513, + "loss": 2.6031, + "theoretical_loss": 3.6062873318065716, + "tokens_seen": 1133969408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315346038114343, + "loss": 2.3994, + "theoretical_loss": 3.6062679561294617, + "tokens_seen": 1134034944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315245737211635, + "loss": 2.7607, + "theoretical_loss": 3.606248581885543, + "tokens_seen": 1134100480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315145436308927, + "loss": 2.7066, + "theoretical_loss": 3.6062292090746277, + "tokens_seen": 1134166016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315045135406219, + "loss": 2.6768, + "theoretical_loss": 3.606209837696527, + "tokens_seen": 1134231552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033149448345035104, + "loss": 2.6928, + "theoretical_loss": 3.6061904677510506, + "tokens_seen": 1134297088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033148445336008027, + "loss": 2.6556, + "theoretical_loss": 3.6061710992380114, + "tokens_seen": 1134362624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314744232698094, + "loss": 2.9638, + "theoretical_loss": 3.60615173215722, + "tokens_seen": 1134428160 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033146439317953863, + "loss": 2.7812, + "theoretical_loss": 3.606132366508488, + "tokens_seen": 1134493696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314543630892678, + "loss": 2.5463, + "theoretical_loss": 3.6061130022916266, + "tokens_seen": 1134559232 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331444332998997, + "loss": 2.8497, + "theoretical_loss": 3.6060936395064473, + "tokens_seen": 1134624768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314343029087262, + "loss": 2.7433, + "theoretical_loss": 3.606074278152762, + "tokens_seen": 1134690304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314242728184554, + "loss": 2.9172, + "theoretical_loss": 3.606054918230382, + "tokens_seen": 1134755840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033141424272818454, + "loss": 2.6835, + "theoretical_loss": 3.6060355597391176, + "tokens_seen": 1134821376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314042126379138, + "loss": 2.6681, + "theoretical_loss": 3.6060162026787825, + "tokens_seen": 1134886912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313941825476429, + "loss": 2.6483, + "theoretical_loss": 3.6059968470491865, + "tokens_seen": 1134952448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033138415245737214, + "loss": 2.723, + "theoretical_loss": 3.6059774928501422, + "tokens_seen": 1135017984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313741223671013, + "loss": 2.6665, + "theoretical_loss": 3.6059581400814613, + "tokens_seen": 1135083520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313640922768305, + "loss": 2.8749, + "theoretical_loss": 3.6059387887429546, + "tokens_seen": 1135149056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313540621865597, + "loss": 2.6138, + "theoretical_loss": 3.605919438834435, + "tokens_seen": 1135214592 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033134403209628886, + "loss": 2.7763, + "theoretical_loss": 3.605900090355714, + "tokens_seen": 1135280128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033133400200601804, + "loss": 2.7442, + "theoretical_loss": 3.605880743306603, + "tokens_seen": 1135345664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 630145, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.701599597930908, + "objective/train/theoretical_loss": 3.605861397686914, + "objective/train/tokens_used": 1155871200, + "theoretical_loss": 3.605861397686914, + "tokens_seen": 1135411200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313239719157473, + "loss": 2.7269, + "theoretical_loss": 3.605861397686914, + "tokens_seen": 1135411200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313139418254764, + "loss": 2.8759, + "theoretical_loss": 3.605842053496459, + "tokens_seen": 1135476736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033130391173520564, + "loss": 2.8524, + "theoretical_loss": 3.60582271073505, + "tokens_seen": 1135542272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033129388164493477, + "loss": 2.7067, + "theoretical_loss": 3.6058033694024987, + "tokens_seen": 1135607808 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331283851554664, + "loss": 2.7671, + "theoretical_loss": 3.605784029498617, + "tokens_seen": 1135673344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312738214643932, + "loss": 2.5566, + "theoretical_loss": 3.605764691023218, + "tokens_seen": 1135738880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033126379137412236, + "loss": 2.6674, + "theoretical_loss": 3.605745353976112, + "tokens_seen": 1135804416 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033125376128385155, + "loss": 3.0017, + "theoretical_loss": 3.605726018357113, + "tokens_seen": 1135869952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312437311935808, + "loss": 2.8305, + "theoretical_loss": 3.6057066841660315, + "tokens_seen": 1135935488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312337011033099, + "loss": 2.8144, + "theoretical_loss": 3.605687351402681, + "tokens_seen": 1136001024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033122367101303914, + "loss": 2.9268, + "theoretical_loss": 3.6056680200668723, + "tokens_seen": 1136066560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033121364092276827, + "loss": 2.7998, + "theoretical_loss": 3.605648690158419, + "tokens_seen": 1136132096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312036108324975, + "loss": 2.8524, + "theoretical_loss": 3.605629361677133, + "tokens_seen": 1136197632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311935807422267, + "loss": 2.7364, + "theoretical_loss": 3.605610034622826, + "tokens_seen": 1136263168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033118355065195587, + "loss": 2.8107, + "theoretical_loss": 3.6055907089953108, + "tokens_seen": 1136328704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033117352056168505, + "loss": 2.7178, + "theoretical_loss": 3.6055713847944, + "tokens_seen": 1136394240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033116349047141423, + "loss": 2.8013, + "theoretical_loss": 3.605552062019906, + "tokens_seen": 1136459776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311534603811434, + "loss": 2.8232, + "theoretical_loss": 3.6055327406716406, + "tokens_seen": 1136525312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033114343029087265, + "loss": 2.6199, + "theoretical_loss": 3.6055134207494177, + "tokens_seen": 1136590848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033113340020060177, + "loss": 2.763, + "theoretical_loss": 3.605494102253048, + "tokens_seen": 1136656384 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331123370110331, + "loss": 2.7053, + "theoretical_loss": 3.6054747851823454, + "tokens_seen": 1136721920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033111334002006013, + "loss": 2.8259, + "theoretical_loss": 3.605455469537122, + "tokens_seen": 1136787456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033110330992978937, + "loss": 2.8541, + "theoretical_loss": 3.6054361553171903, + "tokens_seen": 1136852992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033109327983951855, + "loss": 2.7364, + "theoretical_loss": 3.605416842522364, + "tokens_seen": 1136918528 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033108324974924773, + "loss": 2.7473, + "theoretical_loss": 3.605397531152454, + "tokens_seen": 1136984064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 631187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.756531000137329, + "objective/train/theoretical_loss": 3.6053782212072747, + "objective/train/tokens_used": 1157509600, + "theoretical_loss": 3.6053782212072747, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033107321965897697, + "loss": 2.6511, + "theoretical_loss": 3.6053782212072747, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033106318956870615, + "loss": 2.4748, + "theoretical_loss": 3.605358912686638, + "tokens_seen": 1137115136 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033105315947843533, + "loss": 2.869, + "theoretical_loss": 3.6053396055903564, + "tokens_seen": 1137180672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310431293881645, + "loss": 2.6319, + "theoretical_loss": 3.605320299918244, + "tokens_seen": 1137246208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310330992978937, + "loss": 2.7541, + "theoretical_loss": 3.6053009956701128, + "tokens_seen": 1137311744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310230692076229, + "loss": 2.8495, + "theoretical_loss": 3.605281692845776, + "tokens_seen": 1137377280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310130391173521, + "loss": 2.7188, + "theoretical_loss": 3.605262391445047, + "tokens_seen": 1137442816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033100300902708124, + "loss": 2.6702, + "theoretical_loss": 3.605243091467737, + "tokens_seen": 1137508352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033099297893681047, + "loss": 2.6905, + "theoretical_loss": 3.605223792913661, + "tokens_seen": 1137573888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309829488465396, + "loss": 2.9471, + "theoretical_loss": 3.6052044957826315, + "tokens_seen": 1137639424 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033097291875626883, + "loss": 2.9353, + "theoretical_loss": 3.6051852000744606, + "tokens_seen": 1137704960 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330962888665998, + "loss": 2.6811, + "theoretical_loss": 3.6051659057889633, + "tokens_seen": 1137770496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309528585757272, + "loss": 2.8414, + "theoretical_loss": 3.6051466129259513, + "tokens_seen": 1137836032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309428284854564, + "loss": 2.5861, + "theoretical_loss": 3.6051273214852384, + "tokens_seen": 1137901568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309327983951856, + "loss": 2.8635, + "theoretical_loss": 3.6051080314666377, + "tokens_seen": 1137967104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033092276830491474, + "loss": 2.6327, + "theoretical_loss": 3.6050887428699623, + "tokens_seen": 1138032640 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330912738214644, + "loss": 2.5742, + "theoretical_loss": 3.6050694556950256, + "tokens_seen": 1138098176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309027081243731, + "loss": 2.8405, + "theoretical_loss": 3.6050501699416415, + "tokens_seen": 1138163712 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033089267803410234, + "loss": 2.8262, + "theoretical_loss": 3.6050308856096223, + "tokens_seen": 1138229248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308826479438315, + "loss": 2.6865, + "theoretical_loss": 3.6050116026987826, + "tokens_seen": 1138294784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308726178535607, + "loss": 2.6635, + "theoretical_loss": 3.6049923212089343, + "tokens_seen": 1138360320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308625877632899, + "loss": 2.8313, + "theoretical_loss": 3.6049730411398926, + "tokens_seen": 1138425856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033085255767301906, + "loss": 2.6591, + "theoretical_loss": 3.60495376249147, + "tokens_seen": 1138491392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033084252758274824, + "loss": 2.6289, + "theoretical_loss": 3.604934485263481, + "tokens_seen": 1138556928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003308324974924775, + "loss": 2.7447, + "theoretical_loss": 3.604915209455738, + "tokens_seen": 1138622464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 631834, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4626145362854004, + "objective/train/theoretical_loss": 3.604895935068055, + "objective/train/tokens_used": 1159148000, + "theoretical_loss": 3.604895935068055, + "tokens_seen": 1138688000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003308224674022066, + "loss": 2.8697, + "theoretical_loss": 3.604895935068055, + "tokens_seen": 1138688000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033081243731193584, + "loss": 2.8243, + "theoretical_loss": 3.604876662100246, + "tokens_seen": 1138753536 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033080240722166497, + "loss": 2.742, + "theoretical_loss": 3.604857390552125, + "tokens_seen": 1138819072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307923771313942, + "loss": 2.7061, + "theoretical_loss": 3.604838120423504, + "tokens_seen": 1138884608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307823470411234, + "loss": 2.7016, + "theoretical_loss": 3.6048188517141995, + "tokens_seen": 1138950144 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033077231695085256, + "loss": 2.7084, + "theoretical_loss": 3.604799584424023, + "tokens_seen": 1139015680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033076228686058175, + "loss": 2.7386, + "theoretical_loss": 3.604780318552789, + "tokens_seen": 1139081216 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330752256770311, + "loss": 2.7143, + "theoretical_loss": 3.6047610541003117, + "tokens_seen": 1139146752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307422266800401, + "loss": 2.8785, + "theoretical_loss": 3.604741791066405, + "tokens_seen": 1139212288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033073219658976934, + "loss": 2.7387, + "theoretical_loss": 3.6047225294508825, + "tokens_seen": 1139277824 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033072216649949847, + "loss": 2.6078, + "theoretical_loss": 3.6047032692535588, + "tokens_seen": 1139343360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307121364092277, + "loss": 2.6801, + "theoretical_loss": 3.6046840104742466, + "tokens_seen": 1139408896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307021063189569, + "loss": 2.6139, + "theoretical_loss": 3.6046647531127616, + "tokens_seen": 1139474432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033069207622868607, + "loss": 2.7409, + "theoretical_loss": 3.604645497168917, + "tokens_seen": 1139539968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033068204613841525, + "loss": 2.6384, + "theoretical_loss": 3.6046262426425266, + "tokens_seen": 1139605504 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033067201604814443, + "loss": 2.7318, + "theoretical_loss": 3.604606989533405, + "tokens_seen": 1139671040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306619859578736, + "loss": 2.9143, + "theoretical_loss": 3.6045877378413667, + "tokens_seen": 1139736576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033065195586760285, + "loss": 2.6698, + "theoretical_loss": 3.604568487566225, + "tokens_seen": 1139802112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033064192577733197, + "loss": 2.7636, + "theoretical_loss": 3.6045492387077958, + "tokens_seen": 1139867648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306318956870612, + "loss": 2.7425, + "theoretical_loss": 3.604529991265892, + "tokens_seen": 1139933184 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033062186559679034, + "loss": 2.602, + "theoretical_loss": 3.6045107452403276, + "tokens_seen": 1139998720 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033061183550651957, + "loss": 2.7949, + "theoretical_loss": 3.6044915006309184, + "tokens_seen": 1140064256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033060180541624875, + "loss": 2.6797, + "theoretical_loss": 3.6044722574374775, + "tokens_seen": 1140129792 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033059177532597793, + "loss": 2.5068, + "theoretical_loss": 3.60445301565982, + "tokens_seen": 1140195328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305817452357071, + "loss": 2.5564, + "theoretical_loss": 3.6044337752977604, + "tokens_seen": 1140260864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 632825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7563414573669434, + "objective/train/theoretical_loss": 3.604414536351113, + "objective/train/tokens_used": 1160786400, + "theoretical_loss": 3.604414536351113, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033057171514543635, + "loss": 2.8835, + "theoretical_loss": 3.604414536351113, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305616850551655, + "loss": 2.8897, + "theoretical_loss": 3.604395298819692, + "tokens_seen": 1140391936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305516549648947, + "loss": 2.8563, + "theoretical_loss": 3.604376062703313, + "tokens_seen": 1140457472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033054162487462384, + "loss": 2.7353, + "theoretical_loss": 3.60435682800179, + "tokens_seen": 1140523008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305315947843531, + "loss": 2.7761, + "theoretical_loss": 3.604337594714937, + "tokens_seen": 1140588544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033052156469408225, + "loss": 2.6665, + "theoretical_loss": 3.6043183628425695, + "tokens_seen": 1140654080 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033051153460381144, + "loss": 2.8084, + "theoretical_loss": 3.6042991323845017, + "tokens_seen": 1140719616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305015045135406, + "loss": 2.91, + "theoretical_loss": 3.604279903340549, + "tokens_seen": 1140785152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304914744232698, + "loss": 2.5918, + "theoretical_loss": 3.6042606757105258, + "tokens_seen": 1140850688 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330481444332999, + "loss": 2.7085, + "theoretical_loss": 3.604241449494247, + "tokens_seen": 1140916224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304714142427282, + "loss": 2.6838, + "theoretical_loss": 3.6042222246915276, + "tokens_seen": 1140981760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033046138415245734, + "loss": 2.591, + "theoretical_loss": 3.604203001302182, + "tokens_seen": 1141047296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304513540621866, + "loss": 2.6663, + "theoretical_loss": 3.604183779326026, + "tokens_seen": 1141112832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304413239719157, + "loss": 2.9393, + "theoretical_loss": 3.604164558762873, + "tokens_seen": 1141178368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033043129388164494, + "loss": 2.6918, + "theoretical_loss": 3.6041453396125394, + "tokens_seen": 1141243904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304212637913741, + "loss": 2.6888, + "theoretical_loss": 3.60412612187484, + "tokens_seen": 1141309440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304112337011033, + "loss": 2.7761, + "theoretical_loss": 3.604106905549589, + "tokens_seen": 1141374976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304012036108325, + "loss": 2.8362, + "theoretical_loss": 3.604087690636603, + "tokens_seen": 1141440512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303911735205617, + "loss": 2.6173, + "theoretical_loss": 3.6040684771356957, + "tokens_seen": 1141506048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033038114343029084, + "loss": 2.7558, + "theoretical_loss": 3.604049265046683, + "tokens_seen": 1141571584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303711133400201, + "loss": 2.8015, + "theoretical_loss": 3.60403005436938, + "tokens_seen": 1141637120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303610832497492, + "loss": 2.7054, + "theoretical_loss": 3.604010845103602, + "tokens_seen": 1141702656 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033035105315947844, + "loss": 2.8058, + "theoretical_loss": 3.6039916372491643, + "tokens_seen": 1141768192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303410230692076, + "loss": 2.8196, + "theoretical_loss": 3.6039724308058814, + "tokens_seen": 1141833728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303309929789368, + "loss": 2.581, + "theoretical_loss": 3.60395322577357, + "tokens_seen": 1141899264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 633465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.615844488143921, + "objective/train/theoretical_loss": 3.6039340221520444, + "objective/train/tokens_used": 1162424800, + "theoretical_loss": 3.6039340221520444, + "tokens_seen": 1141964800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033032096288866604, + "loss": 2.6319, + "theoretical_loss": 3.6039340221520444, + "tokens_seen": 1141964800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033031093279839517, + "loss": 2.7764, + "theoretical_loss": 3.6039148199411204, + "tokens_seen": 1142030336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303009027081244, + "loss": 2.768, + "theoretical_loss": 3.6038956191406135, + "tokens_seen": 1142095872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302908726178536, + "loss": 2.7994, + "theoretical_loss": 3.6038764197503395, + "tokens_seen": 1142161408 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033028084252758276, + "loss": 2.818, + "theoretical_loss": 3.603857221770113, + "tokens_seen": 1142226944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033027081243731195, + "loss": 2.7977, + "theoretical_loss": 3.6038380251997504, + "tokens_seen": 1142292480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302607823470412, + "loss": 2.7323, + "theoretical_loss": 3.603818830039067, + "tokens_seen": 1142358016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302507522567703, + "loss": 2.6949, + "theoretical_loss": 3.603799636287878, + "tokens_seen": 1142423552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033024072216649954, + "loss": 2.7476, + "theoretical_loss": 3.6037804439459995, + "tokens_seen": 1142489088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033023069207622867, + "loss": 2.8042, + "theoretical_loss": 3.6037612530132477, + "tokens_seen": 1142554624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302206619859579, + "loss": 2.832, + "theoretical_loss": 3.6037420634894373, + "tokens_seen": 1142620160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302106318956871, + "loss": 2.5317, + "theoretical_loss": 3.6037228753743844, + "tokens_seen": 1142685696 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033020060180541627, + "loss": 2.791, + "theoretical_loss": 3.6037036886679052, + "tokens_seen": 1142751232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033019057171514545, + "loss": 2.543, + "theoretical_loss": 3.603684503369815, + "tokens_seen": 1142816768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033018054162487463, + "loss": 2.9402, + "theoretical_loss": 3.60366531947993, + "tokens_seen": 1142882304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301705115346038, + "loss": 2.9351, + "theoretical_loss": 3.6036461369980657, + "tokens_seen": 1142947840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033016048144433305, + "loss": 2.765, + "theoretical_loss": 3.603626955924039, + "tokens_seen": 1143013376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301504513540622, + "loss": 2.6821, + "theoretical_loss": 3.6036077762576646, + "tokens_seen": 1143078912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301404212637914, + "loss": 2.6081, + "theoretical_loss": 3.603588597998759, + "tokens_seen": 1143144448 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033013039117352054, + "loss": 2.854, + "theoretical_loss": 3.6035694211471387, + "tokens_seen": 1143209984 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033012036108324977, + "loss": 2.6444, + "theoretical_loss": 3.6035502457026194, + "tokens_seen": 1143275520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033011033099297895, + "loss": 2.5662, + "theoretical_loss": 3.603531071665017, + "tokens_seen": 1143341056 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033010030090270813, + "loss": 2.7041, + "theoretical_loss": 3.6035118990341477, + "tokens_seen": 1143406592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300902708124373, + "loss": 2.7002, + "theoretical_loss": 3.6034927278098277, + "tokens_seen": 1143472128 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033008024072216655, + "loss": 2.7704, + "theoretical_loss": 3.6034735579918737, + "tokens_seen": 1143537664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 634749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3306100368499756, + "objective/train/theoretical_loss": 3.6034543895801017, + "objective/train/tokens_used": 1164063200, + "theoretical_loss": 3.6034543895801017, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300702106318957, + "loss": 2.654, + "theoretical_loss": 3.6034543895801017, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300601805416249, + "loss": 2.7415, + "theoretical_loss": 3.603435222574327, + "tokens_seen": 1143668736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033005015045135404, + "loss": 2.6088, + "theoretical_loss": 3.6034160569743676, + "tokens_seen": 1143734272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300401203610833, + "loss": 2.5878, + "theoretical_loss": 3.6033968927800384, + "tokens_seen": 1143799808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033003009027081245, + "loss": 2.7632, + "theoretical_loss": 3.6033777299911565, + "tokens_seen": 1143865344 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033002006018054164, + "loss": 2.9358, + "theoretical_loss": 3.6033585686075384, + "tokens_seen": 1143930880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300100300902708, + "loss": 2.7065, + "theoretical_loss": 3.603339408629, + "tokens_seen": 1143996416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033, + "loss": 2.7274, + "theoretical_loss": 3.6033202500553587, + "tokens_seen": 1144061952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299899699097292, + "loss": 2.6696, + "theoretical_loss": 3.6033010928864297, + "tokens_seen": 1144127488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299799398194584, + "loss": 2.6557, + "theoretical_loss": 3.6032819371220306, + "tokens_seen": 1144193024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032996990972918754, + "loss": 2.838, + "theoretical_loss": 3.603262782761978, + "tokens_seen": 1144258560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299598796389168, + "loss": 2.8272, + "theoretical_loss": 3.6032436298060873, + "tokens_seen": 1144324096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299498495486459, + "loss": 2.7172, + "theoretical_loss": 3.6032244782541767, + "tokens_seen": 1144389632 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032993981945837514, + "loss": 2.7073, + "theoretical_loss": 3.6032053281060623, + "tokens_seen": 1144455168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299297893681043, + "loss": 2.7657, + "theoretical_loss": 3.6031861793615603, + "tokens_seen": 1144520704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299197592778335, + "loss": 2.8131, + "theoretical_loss": 3.6031670320204885, + "tokens_seen": 1144586240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299097291875627, + "loss": 2.7006, + "theoretical_loss": 3.6031478860826627, + "tokens_seen": 1144651776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298996990972919, + "loss": 2.7931, + "theoretical_loss": 3.6031287415479003, + "tokens_seen": 1144717312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032988966900702104, + "loss": 2.7996, + "theoretical_loss": 3.603109598416018, + "tokens_seen": 1144782848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298796389167503, + "loss": 2.4818, + "theoretical_loss": 3.6030904566868323, + "tokens_seen": 1144848384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298696088264794, + "loss": 2.9172, + "theoretical_loss": 3.603071316360161, + "tokens_seen": 1144913920 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032985957873620864, + "loss": 2.7094, + "theoretical_loss": 3.6030521774358206, + "tokens_seen": 1144979456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298495486459378, + "loss": 2.7017, + "theoretical_loss": 3.603033039913628, + "tokens_seen": 1145044992 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329839518555667, + "loss": 2.7211, + "theoretical_loss": 3.6030139037934, + "tokens_seen": 1145110528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298294884653962, + "loss": 2.6337, + "theoretical_loss": 3.602994769074954, + "tokens_seen": 1145176064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 635305, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9339799880981445, + "objective/train/theoretical_loss": 3.6029756357581073, + "objective/train/tokens_used": 1165701600, + "theoretical_loss": 3.6029756357581073, + "tokens_seen": 1145241600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032981945837512537, + "loss": 2.6328, + "theoretical_loss": 3.6029756357581073, + "tokens_seen": 1145241600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032980942828485455, + "loss": 2.7999, + "theoretical_loss": 3.6029565038426767, + "tokens_seen": 1145307136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297993981945838, + "loss": 2.8183, + "theoretical_loss": 3.60293737332848, + "tokens_seen": 1145372672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297893681043129, + "loss": 2.8277, + "theoretical_loss": 3.602918244215333, + "tokens_seen": 1145438208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032977933801404215, + "loss": 2.6376, + "theoretical_loss": 3.602899116503055, + "tokens_seen": 1145503744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297693079237713, + "loss": 2.7554, + "theoretical_loss": 3.6028799901914614, + "tokens_seen": 1145569280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297592778335005, + "loss": 2.8153, + "theoretical_loss": 3.6028608652803706, + "tokens_seen": 1145634816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297492477432297, + "loss": 2.4666, + "theoretical_loss": 3.6028417417695993, + "tokens_seen": 1145700352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032973921765295887, + "loss": 2.726, + "theoretical_loss": 3.6028226196589648, + "tokens_seen": 1145765888 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032972918756268805, + "loss": 2.6854, + "theoretical_loss": 3.6028034989482856, + "tokens_seen": 1145831424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297191574724173, + "loss": 2.5511, + "theoretical_loss": 3.602784379637378, + "tokens_seen": 1145896960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297091273821464, + "loss": 2.547, + "theoretical_loss": 3.60276526172606, + "tokens_seen": 1145962496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032969909729187565, + "loss": 2.7832, + "theoretical_loss": 3.602746145214149, + "tokens_seen": 1146028032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296890672016048, + "loss": 2.6622, + "theoretical_loss": 3.6027270301014624, + "tokens_seen": 1146093568 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329679037111334, + "loss": 2.6686, + "theoretical_loss": 3.602707916387818, + "tokens_seen": 1146159104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296690070210632, + "loss": 2.7567, + "theoretical_loss": 3.6026888040730336, + "tokens_seen": 1146224640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296589769307924, + "loss": 2.6816, + "theoretical_loss": 3.602669693156926, + "tokens_seen": 1146290176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032964894684052155, + "loss": 2.829, + "theoretical_loss": 3.6026505836393143, + "tokens_seen": 1146355712 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032963891675025074, + "loss": 2.7428, + "theoretical_loss": 3.602631475520015, + "tokens_seen": 1146421248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296288866599799, + "loss": 2.6902, + "theoretical_loss": 3.6026123687988463, + "tokens_seen": 1146486784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032961885656970915, + "loss": 2.7686, + "theoretical_loss": 3.602593263475626, + "tokens_seen": 1146552320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296088264794383, + "loss": 2.7541, + "theoretical_loss": 3.602574159550172, + "tokens_seen": 1146617856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295987963891675, + "loss": 2.4678, + "theoretical_loss": 3.6025550570223013, + "tokens_seen": 1146683392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295887662988967, + "loss": 2.6954, + "theoretical_loss": 3.6025359558918333, + "tokens_seen": 1146748928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295787362086259, + "loss": 2.6374, + "theoretical_loss": 3.6025168561585845, + "tokens_seen": 1146814464 + }, + { + "debugging/Self-BLEU-5": 0.4786247835242382, + "debugging/distinct-1-grams": 0.7609861614906261, + "debugging/distinct-2-grams": 0.9220814316505062, + "debugging/entropy-1-grams": 5.880984152522802, + "debugging/entropy-2-grams": 6.5464310105420465, + "debugging/length": 573.6, + "debugging/num_segments": 10, + "epoch": 0.35, + "objective/train/docs_used": 636617, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.868852376937866, + "objective/train/theoretical_loss": 3.6024977578223742, + "objective/train/tokens_used": 1167340000, + "theoretical_loss": 3.6024977578223742, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295687061183551, + "loss": 2.8676, + "theoretical_loss": 3.6024977578223742, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032955867602808424, + "loss": 2.6701, + "theoretical_loss": 3.6024786608830195, + "tokens_seen": 1146945536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295486459378135, + "loss": 2.9224, + "theoretical_loss": 3.6024595653403386, + "tokens_seen": 1147011072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032953861584754265, + "loss": 2.8055, + "theoretical_loss": 3.602440471194149, + "tokens_seen": 1147076608 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032952858575727184, + "loss": 2.686, + "theoretical_loss": 3.6024213784442702, + "tokens_seen": 1147142144 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329518555667001, + "loss": 2.8341, + "theoretical_loss": 3.602402287090519, + "tokens_seen": 1147207680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295085255767302, + "loss": 2.8485, + "theoretical_loss": 3.6023831971327143, + "tokens_seen": 1147273216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294984954864594, + "loss": 2.6909, + "theoretical_loss": 3.602364108570674, + "tokens_seen": 1147338752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294884653961886, + "loss": 2.7537, + "theoretical_loss": 3.6023450214042163, + "tokens_seen": 1147404288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032947843530591774, + "loss": 2.8729, + "theoretical_loss": 3.60232593563316, + "tokens_seen": 1147469824 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329468405215647, + "loss": 2.9974, + "theoretical_loss": 3.602306851257322, + "tokens_seen": 1147535360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294583751253761, + "loss": 2.6533, + "theoretical_loss": 3.602287768276523, + "tokens_seen": 1147600896 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032944834503510534, + "loss": 2.793, + "theoretical_loss": 3.6022686866905786, + "tokens_seen": 1147666432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294383149448345, + "loss": 2.6353, + "theoretical_loss": 3.602249606499309, + "tokens_seen": 1147731968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294282848545637, + "loss": 2.8417, + "theoretical_loss": 3.602230527702533, + "tokens_seen": 1147797504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294182547642929, + "loss": 2.7079, + "theoretical_loss": 3.602211450300067, + "tokens_seen": 1147863040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294082246740221, + "loss": 2.8198, + "theoretical_loss": 3.6021923742917314, + "tokens_seen": 1147928576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032939819458375124, + "loss": 2.6095, + "theoretical_loss": 3.602173299677344, + "tokens_seen": 1147994112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293881644934805, + "loss": 2.7788, + "theoretical_loss": 3.6021542264567237, + "tokens_seen": 1148059648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293781344032096, + "loss": 2.7369, + "theoretical_loss": 3.6021351546296887, + "tokens_seen": 1148125184 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032936810431293884, + "loss": 2.6818, + "theoretical_loss": 3.6021160841960578, + "tokens_seen": 1148190720 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329358074222668, + "loss": 2.8319, + "theoretical_loss": 3.6020970151556497, + "tokens_seen": 1148256256 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293480441323972, + "loss": 2.6298, + "theoretical_loss": 3.602077947508283, + "tokens_seen": 1148321792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293380140421264, + "loss": 2.7902, + "theoretical_loss": 3.602058881253777, + "tokens_seen": 1148387328 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032932798395185557, + "loss": 2.7109, + "theoretical_loss": 3.6020398163919496, + "tokens_seen": 1148452864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 637752, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6599230766296387, + "objective/train/theoretical_loss": 3.6020207529226203, + "objective/train/tokens_used": 1168978400, + "theoretical_loss": 3.6020207529226203, + "tokens_seen": 1148518400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032931795386158475, + "loss": 2.7556, + "theoretical_loss": 3.6020207529226203, + "tokens_seen": 1148518400 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329307923771314, + "loss": 2.7028, + "theoretical_loss": 3.6020016908456074, + "tokens_seen": 1148583936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292978936810431, + "loss": 2.6435, + "theoretical_loss": 3.6019826301607303, + "tokens_seen": 1148649472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032928786359077235, + "loss": 2.7049, + "theoretical_loss": 3.6019635708678077, + "tokens_seen": 1148715008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292778335005015, + "loss": 2.7472, + "theoretical_loss": 3.601944512966658, + "tokens_seen": 1148780544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292678034102307, + "loss": 2.7465, + "theoretical_loss": 3.601925456457101, + "tokens_seen": 1148846080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292577733199599, + "loss": 2.5858, + "theoretical_loss": 3.6019064013389555, + "tokens_seen": 1148911616 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032924774322968907, + "loss": 2.8702, + "theoretical_loss": 3.60188734761204, + "tokens_seen": 1148977152 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032923771313941825, + "loss": 2.6107, + "theoretical_loss": 3.6018682952761747, + "tokens_seen": 1149042688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292276830491475, + "loss": 2.8954, + "theoretical_loss": 3.6018492443311776, + "tokens_seen": 1149108224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292176529588766, + "loss": 2.5675, + "theoretical_loss": 3.6018301947768685, + "tokens_seen": 1149173760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032920762286860585, + "loss": 2.7175, + "theoretical_loss": 3.6018111466130662, + "tokens_seen": 1149239296 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329197592778335, + "loss": 2.7522, + "theoretical_loss": 3.6017920998395896, + "tokens_seen": 1149304832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291875626880642, + "loss": 2.6172, + "theoretical_loss": 3.6017730544562596, + "tokens_seen": 1149370368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291775325977934, + "loss": 2.6186, + "theoretical_loss": 3.6017540104628933, + "tokens_seen": 1149435904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291675025075226, + "loss": 2.8122, + "theoretical_loss": 3.6017349678593114, + "tokens_seen": 1149501440 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032915747241725175, + "loss": 2.5775, + "theoretical_loss": 3.601715926645333, + "tokens_seen": 1149566976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032914744232698094, + "loss": 2.7746, + "theoretical_loss": 3.6016968868207773, + "tokens_seen": 1149632512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291374122367101, + "loss": 2.5421, + "theoretical_loss": 3.6016778483854637, + "tokens_seen": 1149698048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032912738214643935, + "loss": 2.4038, + "theoretical_loss": 3.6016588113392114, + "tokens_seen": 1149763584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291173520561685, + "loss": 2.7402, + "theoretical_loss": 3.601639775681841, + "tokens_seen": 1149829120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291073219658977, + "loss": 2.7332, + "theoretical_loss": 3.6016207414131705, + "tokens_seen": 1149894656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290972918756269, + "loss": 2.7302, + "theoretical_loss": 3.6016017085330203, + "tokens_seen": 1149960192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290872617853561, + "loss": 2.661, + "theoretical_loss": 3.6015826770412103, + "tokens_seen": 1150025728 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032907723169508526, + "loss": 2.7305, + "theoretical_loss": 3.601563646937559, + "tokens_seen": 1150091264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 638347, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3350415229797363, + "objective/train/theoretical_loss": 3.6015446182218875, + "objective/train/tokens_used": 1170616800, + "theoretical_loss": 3.6015446182218875, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032906720160481444, + "loss": 2.6234, + "theoretical_loss": 3.6015446182218875, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290571715145436, + "loss": 2.5445, + "theoretical_loss": 3.6015255908940147, + "tokens_seen": 1150222336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032904714142427286, + "loss": 2.6305, + "theoretical_loss": 3.60150656495376, + "tokens_seen": 1150287872 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329037111334002, + "loss": 2.7704, + "theoretical_loss": 3.6014875404009437, + "tokens_seen": 1150353408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290270812437312, + "loss": 2.8285, + "theoretical_loss": 3.601468517235385, + "tokens_seen": 1150418944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032901705115346034, + "loss": 2.7081, + "theoretical_loss": 3.6014494954569045, + "tokens_seen": 1150484480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290070210631896, + "loss": 2.7436, + "theoretical_loss": 3.601430475065322, + "tokens_seen": 1150550016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032899699097291876, + "loss": 2.7384, + "theoretical_loss": 3.6014114560604567, + "tokens_seen": 1150615552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032898696088264794, + "loss": 2.7074, + "theoretical_loss": 3.6013924384421285, + "tokens_seen": 1150681088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289769307923771, + "loss": 2.671, + "theoretical_loss": 3.6013734222101585, + "tokens_seen": 1150746624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289669007021063, + "loss": 2.6452, + "theoretical_loss": 3.6013544073643655, + "tokens_seen": 1150812160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289568706118355, + "loss": 2.5962, + "theoretical_loss": 3.60133539390457, + "tokens_seen": 1150877696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289468405215647, + "loss": 2.8025, + "theoretical_loss": 3.601316381830592, + "tokens_seen": 1150943232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032893681043129385, + "loss": 2.6795, + "theoretical_loss": 3.601297371142252, + "tokens_seen": 1151008768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289267803410231, + "loss": 2.7709, + "theoretical_loss": 3.6012783618393693, + "tokens_seen": 1151074304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032891675025075226, + "loss": 2.7108, + "theoretical_loss": 3.601259353921765, + "tokens_seen": 1151139840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032890672016048145, + "loss": 2.8169, + "theoretical_loss": 3.6012403473892585, + "tokens_seen": 1151205376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288966900702106, + "loss": 2.7462, + "theoretical_loss": 3.6012213422416703, + "tokens_seen": 1151270912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288866599799398, + "loss": 2.6478, + "theoretical_loss": 3.6012023384788208, + "tokens_seen": 1151336448 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328876629889669, + "loss": 2.7033, + "theoretical_loss": 3.60118333610053, + "tokens_seen": 1151401984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288665997993982, + "loss": 2.7022, + "theoretical_loss": 3.6011643351066183, + "tokens_seen": 1151467520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032885656970912735, + "loss": 2.8157, + "theoretical_loss": 3.6011453354969065, + "tokens_seen": 1151533056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288465396188566, + "loss": 2.6348, + "theoretical_loss": 3.6011263372712143, + "tokens_seen": 1151598592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288365095285857, + "loss": 2.7876, + "theoretical_loss": 3.6011073404293628, + "tokens_seen": 1151664128 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032882647943831495, + "loss": 2.5848, + "theoretical_loss": 3.601088344971172, + "tokens_seen": 1151729664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 639523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5807628631591797, + "objective/train/theoretical_loss": 3.6010693508964624, + "objective/train/tokens_used": 1172255200, + "theoretical_loss": 3.6010693508964624, + "tokens_seen": 1151795200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288164493480442, + "loss": 2.5903, + "theoretical_loss": 3.6010693508964624, + "tokens_seen": 1151795200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288064192577733, + "loss": 2.5482, + "theoretical_loss": 3.601050358205055, + "tokens_seen": 1151860736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032879638916750255, + "loss": 2.7664, + "theoretical_loss": 3.6010313668967697, + "tokens_seen": 1151926272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287863590772317, + "loss": 2.7388, + "theoretical_loss": 3.6010123769714273, + "tokens_seen": 1151991808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287763289869609, + "loss": 2.7822, + "theoretical_loss": 3.600993388428849, + "tokens_seen": 1152057344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287662988966901, + "loss": 2.8795, + "theoretical_loss": 3.600974401268855, + "tokens_seen": 1152122880 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032875626880641927, + "loss": 2.5308, + "theoretical_loss": 3.600955415491266, + "tokens_seen": 1152188416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032874623871614845, + "loss": 2.9124, + "theoretical_loss": 3.6009364310959024, + "tokens_seen": 1152253952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287362086258777, + "loss": 2.7283, + "theoretical_loss": 3.6009174480825856, + "tokens_seen": 1152319488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287261785356068, + "loss": 2.7157, + "theoretical_loss": 3.6008984664511363, + "tokens_seen": 1152385024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032871614844533605, + "loss": 2.5741, + "theoretical_loss": 3.6008794862013747, + "tokens_seen": 1152450560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287061183550652, + "loss": 2.9161, + "theoretical_loss": 3.600860507333122, + "tokens_seen": 1152516096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286960882647944, + "loss": 2.6876, + "theoretical_loss": 3.6008415298461993, + "tokens_seen": 1152581632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286860581745236, + "loss": 2.7743, + "theoretical_loss": 3.6008225537404277, + "tokens_seen": 1152647168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286760280842528, + "loss": 2.8304, + "theoretical_loss": 3.6008035790156274, + "tokens_seen": 1152712704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032866599799398195, + "loss": 2.9008, + "theoretical_loss": 3.6007846056716204, + "tokens_seen": 1152778240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032865596790371114, + "loss": 2.7073, + "theoretical_loss": 3.6007656337082268, + "tokens_seen": 1152843776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286459378134403, + "loss": 2.4973, + "theoretical_loss": 3.6007466631252685, + "tokens_seen": 1152909312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032863590772316955, + "loss": 2.6311, + "theoretical_loss": 3.6007276939225656, + "tokens_seen": 1152974848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286258776328987, + "loss": 2.5766, + "theoretical_loss": 3.60070872609994, + "tokens_seen": 1153040384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286158475426279, + "loss": 2.5913, + "theoretical_loss": 3.6006897596572127, + "tokens_seen": 1153105920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286058174523571, + "loss": 2.5173, + "theoretical_loss": 3.6006707945942047, + "tokens_seen": 1153171456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285957873620863, + "loss": 2.6512, + "theoretical_loss": 3.6006518309107376, + "tokens_seen": 1153236992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032858575727181546, + "loss": 2.8246, + "theoretical_loss": 3.6006328686066325, + "tokens_seen": 1153302528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032857572718154464, + "loss": 2.8009, + "theoretical_loss": 3.6006139076817103, + "tokens_seen": 1153368064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 639983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.798460006713867, + "objective/train/theoretical_loss": 3.600594948135793, + "objective/train/tokens_used": 1173893600, + "theoretical_loss": 3.600594948135793, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285656970912738, + "loss": 2.6424, + "theoretical_loss": 3.600594948135793, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032855566700100306, + "loss": 2.6186, + "theoretical_loss": 3.6005759899687018, + "tokens_seen": 1153499136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285456369107322, + "loss": 2.8315, + "theoretical_loss": 3.6005570331802574, + "tokens_seen": 1153564672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285356068204614, + "loss": 2.7475, + "theoretical_loss": 3.600538077770282, + "tokens_seen": 1153630208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032852557673019054, + "loss": 2.7912, + "theoretical_loss": 3.6005191237385965, + "tokens_seen": 1153695744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285155466399198, + "loss": 2.6463, + "theoretical_loss": 3.6005001710850233, + "tokens_seen": 1153761280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032850551654964896, + "loss": 2.5891, + "theoretical_loss": 3.6004812198093834, + "tokens_seen": 1153826816 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032849548645937814, + "loss": 2.4599, + "theoretical_loss": 3.600462269911498, + "tokens_seen": 1153892352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284854563691073, + "loss": 2.6544, + "theoretical_loss": 3.6004433213911886, + "tokens_seen": 1153957888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284754262788365, + "loss": 2.5024, + "theoretical_loss": 3.6004243742482775, + "tokens_seen": 1154023424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284653961885657, + "loss": 2.5078, + "theoretical_loss": 3.600405428482586, + "tokens_seen": 1154088960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284553660982949, + "loss": 2.6429, + "theoretical_loss": 3.600386484093936, + "tokens_seen": 1154154496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032844533600802405, + "loss": 2.6753, + "theoretical_loss": 3.600367541082149, + "tokens_seen": 1154220032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284353059177533, + "loss": 2.6927, + "theoretical_loss": 3.6003485994470474, + "tokens_seen": 1154285568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032842527582748246, + "loss": 2.5926, + "theoretical_loss": 3.600329659188452, + "tokens_seen": 1154351104 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032841524573721165, + "loss": 2.6645, + "theoretical_loss": 3.6003107203061853, + "tokens_seen": 1154416640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284052156469408, + "loss": 2.8488, + "theoretical_loss": 3.6002917828000687, + "tokens_seen": 1154482176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032839518555667, + "loss": 2.6092, + "theoretical_loss": 3.6002728466699248, + "tokens_seen": 1154547712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283851554663992, + "loss": 2.67, + "theoretical_loss": 3.6002539119155745, + "tokens_seen": 1154613248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283751253761284, + "loss": 2.57, + "theoretical_loss": 3.6002349785368413, + "tokens_seen": 1154678784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032836509528585755, + "loss": 2.594, + "theoretical_loss": 3.6002160465335455, + "tokens_seen": 1154744320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283550651955868, + "loss": 2.8139, + "theoretical_loss": 3.6001971159055097, + "tokens_seen": 1154809856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283450351053159, + "loss": 2.6035, + "theoretical_loss": 3.600178186652556, + "tokens_seen": 1154875392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032833500501504515, + "loss": 2.8044, + "theoretical_loss": 3.6001592587745073, + "tokens_seen": 1154940928 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032832497492477433, + "loss": 2.6125, + "theoretical_loss": 3.6001403322711845, + "tokens_seen": 1155006464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 641200, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4352283477783203, + "objective/train/theoretical_loss": 3.6001214071424106, + "objective/train/tokens_used": 1175532000, + "theoretical_loss": 3.6001214071424106, + "tokens_seen": 1155072000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283149448345035, + "loss": 2.718, + "theoretical_loss": 3.6001214071424106, + "tokens_seen": 1155072000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283049147442327, + "loss": 2.7494, + "theoretical_loss": 3.6001024833880075, + "tokens_seen": 1155137536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282948846539619, + "loss": 2.7, + "theoretical_loss": 3.6000835610077973, + "tokens_seen": 1155203072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032828485456369105, + "loss": 2.6363, + "theoretical_loss": 3.6000646400016025, + "tokens_seen": 1155268608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282748244734203, + "loss": 2.6263, + "theoretical_loss": 3.6000457203692453, + "tokens_seen": 1155334144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282647943831494, + "loss": 2.5974, + "theoretical_loss": 3.6000268021105475, + "tokens_seen": 1155399680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032825476429287865, + "loss": 2.9712, + "theoretical_loss": 3.6000078852253328, + "tokens_seen": 1155465216 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032824473420260783, + "loss": 2.5518, + "theoretical_loss": 3.599988969713422, + "tokens_seen": 1155530752 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328234704112337, + "loss": 2.6989, + "theoretical_loss": 3.5999700555746386, + "tokens_seen": 1155596288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282246740220662, + "loss": 2.7771, + "theoretical_loss": 3.5999511428088047, + "tokens_seen": 1155661824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282146439317954, + "loss": 2.6778, + "theoretical_loss": 3.5999322314157425, + "tokens_seen": 1155727360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032820461384152456, + "loss": 2.5627, + "theoretical_loss": 3.5999133213952756, + "tokens_seen": 1155792896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281945837512538, + "loss": 2.9634, + "theoretical_loss": 3.599894412747225, + "tokens_seen": 1155858432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281845536609829, + "loss": 2.8353, + "theoretical_loss": 3.599875505471415, + "tokens_seen": 1155923968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032817452357071215, + "loss": 2.9044, + "theoretical_loss": 3.599856599567666, + "tokens_seen": 1155989504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281644934804413, + "loss": 2.6862, + "theoretical_loss": 3.599837695035803, + "tokens_seen": 1156055040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281544633901705, + "loss": 2.8285, + "theoretical_loss": 3.5998187918756472, + "tokens_seen": 1156120576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281444332998997, + "loss": 2.8731, + "theoretical_loss": 3.599799890087022, + "tokens_seen": 1156186112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281344032096289, + "loss": 2.6179, + "theoretical_loss": 3.5997809896697497, + "tokens_seen": 1156251648 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032812437311935806, + "loss": 2.5605, + "theoretical_loss": 3.5997620906236536, + "tokens_seen": 1156317184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281143430290873, + "loss": 2.6217, + "theoretical_loss": 3.599743192948556, + "tokens_seen": 1156382720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281043129388164, + "loss": 2.7596, + "theoretical_loss": 3.59972429664428, + "tokens_seen": 1156448256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032809428284854566, + "loss": 2.5259, + "theoretical_loss": 3.599705401710648, + "tokens_seen": 1156513792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280842527582748, + "loss": 2.473, + "theoretical_loss": 3.5996865081474843, + "tokens_seen": 1156579328 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328074222668004, + "loss": 2.6603, + "theoretical_loss": 3.59966761595461, + "tokens_seen": 1156644864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 641801, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.768510341644287, + "objective/train/theoretical_loss": 3.59964872513185, + "objective/train/tokens_used": 1177170400, + "theoretical_loss": 3.59964872513185, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032806419257773326, + "loss": 2.7684, + "theoretical_loss": 3.59964872513185, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280541624874624, + "loss": 2.7661, + "theoretical_loss": 3.5996298356790257, + "tokens_seen": 1156775936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280441323971916, + "loss": 2.5801, + "theoretical_loss": 3.599610947595961, + "tokens_seen": 1156841472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032803410230692074, + "loss": 2.8947, + "theoretical_loss": 3.599592060882479, + "tokens_seen": 1156907008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032802407221665, + "loss": 2.8118, + "theoretical_loss": 3.599573175538402, + "tokens_seen": 1156972544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032801404212637916, + "loss": 2.713, + "theoretical_loss": 3.599554291563554, + "tokens_seen": 1157038080 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032800401203610834, + "loss": 2.6654, + "theoretical_loss": 3.5995354089577583, + "tokens_seen": 1157103616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279939819458375, + "loss": 2.4662, + "theoretical_loss": 3.599516527720837, + "tokens_seen": 1157169152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279839518555667, + "loss": 2.7589, + "theoretical_loss": 3.5994976478526146, + "tokens_seen": 1157234688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279739217652959, + "loss": 2.7968, + "theoretical_loss": 3.599478769352914, + "tokens_seen": 1157300224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279638916750251, + "loss": 2.5783, + "theoretical_loss": 3.5994598922215584, + "tokens_seen": 1157365760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032795386158475425, + "loss": 2.662, + "theoretical_loss": 3.5994410164583703, + "tokens_seen": 1157431296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279438314944835, + "loss": 2.5359, + "theoretical_loss": 3.5994221420631747, + "tokens_seen": 1157496832 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032793380140421266, + "loss": 2.7496, + "theoretical_loss": 3.5994032690357938, + "tokens_seen": 1157562368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032792377131394185, + "loss": 2.7392, + "theoretical_loss": 3.5993843973760518, + "tokens_seen": 1157627904 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327913741223671, + "loss": 2.6497, + "theoretical_loss": 3.599365527083772, + "tokens_seen": 1157693440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279037111334002, + "loss": 2.6444, + "theoretical_loss": 3.599346658158777, + "tokens_seen": 1157758976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278936810431294, + "loss": 2.8481, + "theoretical_loss": 3.599327790600891, + "tokens_seen": 1157824512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278836509528586, + "loss": 2.7322, + "theoretical_loss": 3.5993089244099385, + "tokens_seen": 1157890048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032787362086258775, + "loss": 2.6314, + "theoretical_loss": 3.5992900595857416, + "tokens_seen": 1157955584 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327863590772317, + "loss": 2.5972, + "theoretical_loss": 3.599271196128125, + "tokens_seen": 1158021120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278535606820461, + "loss": 2.6548, + "theoretical_loss": 3.599252334036912, + "tokens_seen": 1158086656 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032784353059177535, + "loss": 2.5857, + "theoretical_loss": 3.5992334733119264, + "tokens_seen": 1158152192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032783350050150453, + "loss": 2.7087, + "theoretical_loss": 3.5992146139529915, + "tokens_seen": 1158217728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278234704112337, + "loss": 2.5285, + "theoretical_loss": 3.599195755959931, + "tokens_seen": 1158283264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 642800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7663416862487793, + "objective/train/theoretical_loss": 3.59917689933257, + "objective/train/tokens_used": 1178808800, + "theoretical_loss": 3.59917689933257, + "tokens_seen": 1158348800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278134403209629, + "loss": 2.7091, + "theoretical_loss": 3.59917689933257, + "tokens_seen": 1158348800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032780341023069213, + "loss": 2.6744, + "theoretical_loss": 3.599158044070731, + "tokens_seen": 1158414336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032779338014042125, + "loss": 2.5622, + "theoretical_loss": 3.5991391901742382, + "tokens_seen": 1158479872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277833500501505, + "loss": 2.6717, + "theoretical_loss": 3.599120337642916, + "tokens_seen": 1158545408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277733199598796, + "loss": 2.5785, + "theoretical_loss": 3.599101486476587, + "tokens_seen": 1158610944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032776328986960885, + "loss": 2.5764, + "theoretical_loss": 3.599082636675077, + "tokens_seen": 1158676480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032775325977933803, + "loss": 2.6596, + "theoretical_loss": 3.599063788238209, + "tokens_seen": 1158742016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277432296890672, + "loss": 2.6874, + "theoretical_loss": 3.599044941165807, + "tokens_seen": 1158807552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277331995987964, + "loss": 2.8803, + "theoretical_loss": 3.5990260954576954, + "tokens_seen": 1158873088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277231695085256, + "loss": 2.793, + "theoretical_loss": 3.599007251113698, + "tokens_seen": 1158938624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032771313941825476, + "loss": 2.7827, + "theoretical_loss": 3.598988408133639, + "tokens_seen": 1159004160 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327703109327984, + "loss": 2.7677, + "theoretical_loss": 3.5989695665173427, + "tokens_seen": 1159069696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276930792377131, + "loss": 2.5865, + "theoretical_loss": 3.598950726264633, + "tokens_seen": 1159135232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032768304914744235, + "loss": 2.6556, + "theoretical_loss": 3.5989318873753344, + "tokens_seen": 1159200768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276730190571715, + "loss": 2.8157, + "theoretical_loss": 3.5989130498492714, + "tokens_seen": 1159266304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276629889669007, + "loss": 2.6851, + "theoretical_loss": 3.5988942136862674, + "tokens_seen": 1159331840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276529588766299, + "loss": 2.7396, + "theoretical_loss": 3.598875378886148, + "tokens_seen": 1159397376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276429287863591, + "loss": 2.8136, + "theoretical_loss": 3.5988565454487365, + "tokens_seen": 1159462912 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032763289869608826, + "loss": 2.8422, + "theoretical_loss": 3.598837713373858, + "tokens_seen": 1159528448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276228686058175, + "loss": 2.6567, + "theoretical_loss": 3.598818882661336, + "tokens_seen": 1159593984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276128385155466, + "loss": 2.694, + "theoretical_loss": 3.598800053310996, + "tokens_seen": 1159659520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032760280842527586, + "loss": 2.7281, + "theoretical_loss": 3.5987812253226616, + "tokens_seen": 1159725056 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327592778335005, + "loss": 2.7379, + "theoretical_loss": 3.598762398696158, + "tokens_seen": 1159790592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275827482447342, + "loss": 2.8922, + "theoretical_loss": 3.5987435734313094, + "tokens_seen": 1159856128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275727181544634, + "loss": 2.5817, + "theoretical_loss": 3.598724749527941, + "tokens_seen": 1159921664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 643907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6200287342071533, + "objective/train/theoretical_loss": 3.5987059269858763, + "objective/train/tokens_used": 1180447200, + "theoretical_loss": 3.5987059269858763, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275626880641926, + "loss": 2.7223, + "theoretical_loss": 3.5987059269858763, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032755265797392176, + "loss": 2.7649, + "theoretical_loss": 3.5986871058049408, + "tokens_seen": 1160052736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032754262788365094, + "loss": 2.5639, + "theoretical_loss": 3.598668285984959, + "tokens_seen": 1160118272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275325977933801, + "loss": 2.7138, + "theoretical_loss": 3.5986494675257554, + "tokens_seen": 1160183808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032752256770310936, + "loss": 2.6548, + "theoretical_loss": 3.598630650427155, + "tokens_seen": 1160249344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275125376128385, + "loss": 2.6848, + "theoretical_loss": 3.598611834688983, + "tokens_seen": 1160314880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275025075225677, + "loss": 2.7612, + "theoretical_loss": 3.5985930203110628, + "tokens_seen": 1160380416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032749247743229685, + "loss": 2.9665, + "theoretical_loss": 3.598574207293221, + "tokens_seen": 1160445952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274824473420261, + "loss": 2.8244, + "theoretical_loss": 3.598555395635281, + "tokens_seen": 1160511488 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032747241725175527, + "loss": 2.5306, + "theoretical_loss": 3.5985365853370688, + "tokens_seen": 1160577024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032746238716148445, + "loss": 2.7923, + "theoretical_loss": 3.5985177763984084, + "tokens_seen": 1160642560 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032745235707121363, + "loss": 2.756, + "theoretical_loss": 3.5984989688191256, + "tokens_seen": 1160708096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032744232698094286, + "loss": 2.6654, + "theoretical_loss": 3.598480162599045, + "tokens_seen": 1160773632 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327432296890672, + "loss": 2.8767, + "theoretical_loss": 3.5984613577379916, + "tokens_seen": 1160839168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274222668004012, + "loss": 2.6347, + "theoretical_loss": 3.598442554235791, + "tokens_seen": 1160904704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032741223671013035, + "loss": 2.6026, + "theoretical_loss": 3.598423752092268, + "tokens_seen": 1160970240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274022066198596, + "loss": 2.5634, + "theoretical_loss": 3.5984049513072467, + "tokens_seen": 1161035776 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032739217652958877, + "loss": 2.6826, + "theoretical_loss": 3.598386151880554, + "tokens_seen": 1161101312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032738214643931795, + "loss": 2.8304, + "theoretical_loss": 3.598367353812014, + "tokens_seen": 1161166848 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032737211634904713, + "loss": 2.6202, + "theoretical_loss": 3.5983485571014526, + "tokens_seen": 1161232384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273620862587763, + "loss": 2.5217, + "theoretical_loss": 3.5983297617486945, + "tokens_seen": 1161297920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273520561685055, + "loss": 2.6864, + "theoretical_loss": 3.598310967753565, + "tokens_seen": 1161363456 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032734202607823473, + "loss": 2.776, + "theoretical_loss": 3.59829217511589, + "tokens_seen": 1161428992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032733199598796386, + "loss": 2.6287, + "theoretical_loss": 3.5982733838354948, + "tokens_seen": 1161494528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273219658976931, + "loss": 2.5982, + "theoretical_loss": 3.5982545939122037, + "tokens_seen": 1161560064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 644495, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7944414615631104, + "objective/train/theoretical_loss": 3.598235805345843, + "objective/train/tokens_used": 1182085600, + "theoretical_loss": 3.598235805345843, + "tokens_seen": 1161625600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032731193580742233, + "loss": 2.7485, + "theoretical_loss": 3.598235805345843, + "tokens_seen": 1161625600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032730190571715145, + "loss": 2.7042, + "theoretical_loss": 3.598217018136239, + "tokens_seen": 1161691136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272918756268807, + "loss": 2.6849, + "theoretical_loss": 3.598198232283216, + "tokens_seen": 1161756672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272818455366098, + "loss": 2.5112, + "theoretical_loss": 3.598179447786599, + "tokens_seen": 1161822208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032727181544633905, + "loss": 2.5599, + "theoretical_loss": 3.598160664646215, + "tokens_seen": 1161887744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032726178535606823, + "loss": 2.4779, + "theoretical_loss": 3.598141882861889, + "tokens_seen": 1161953280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272517552657974, + "loss": 2.508, + "theoretical_loss": 3.598123102433447, + "tokens_seen": 1162018816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272417251755266, + "loss": 2.7165, + "theoretical_loss": 3.5981043233607135, + "tokens_seen": 1162084352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272316950852558, + "loss": 2.6167, + "theoretical_loss": 3.5980855456435155, + "tokens_seen": 1162149888 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032722166499498496, + "loss": 2.6884, + "theoretical_loss": 3.5980667692816777, + "tokens_seen": 1162215424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272116349047142, + "loss": 2.5378, + "theoretical_loss": 3.598047994275026, + "tokens_seen": 1162280960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272016048144433, + "loss": 2.6135, + "theoretical_loss": 3.5980292206233875, + "tokens_seen": 1162346496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032719157472417256, + "loss": 2.6741, + "theoretical_loss": 3.598010448326586, + "tokens_seen": 1162412032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271815446339017, + "loss": 2.7771, + "theoretical_loss": 3.5979916773844494, + "tokens_seen": 1162477568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271715145436309, + "loss": 2.5643, + "theoretical_loss": 3.5979729077968017, + "tokens_seen": 1162543104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271614844533601, + "loss": 2.6286, + "theoretical_loss": 3.5979541395634698, + "tokens_seen": 1162608640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271514543630893, + "loss": 2.7594, + "theoretical_loss": 3.59793537268428, + "tokens_seen": 1162674176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032714142427281846, + "loss": 2.5682, + "theoretical_loss": 3.5979166071590574, + "tokens_seen": 1162739712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271313941825477, + "loss": 2.8591, + "theoretical_loss": 3.5978978429876283, + "tokens_seen": 1162805248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271213640922768, + "loss": 2.5559, + "theoretical_loss": 3.597879080169819, + "tokens_seen": 1162870784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032711133400200606, + "loss": 2.6989, + "theoretical_loss": 3.5978603187054556, + "tokens_seen": 1162936320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271013039117352, + "loss": 2.4975, + "theoretical_loss": 3.5978415585943635, + "tokens_seen": 1163001856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270912738214644, + "loss": 2.7157, + "theoretical_loss": 3.5978227998363694, + "tokens_seen": 1163067392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270812437311936, + "loss": 2.5283, + "theoretical_loss": 3.5978040424313003, + "tokens_seen": 1163132928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270712136409228, + "loss": 2.7726, + "theoretical_loss": 3.5977852863789805, + "tokens_seen": 1163198464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 645795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8706414699554443, + "objective/train/theoretical_loss": 3.5977665316792375, + "objective/train/tokens_used": 1183724000, + "theoretical_loss": 3.5977665316792375, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032706118355065196, + "loss": 2.708, + "theoretical_loss": 3.5977665316792375, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032705115346038114, + "loss": 2.54, + "theoretical_loss": 3.5977477783318976, + "tokens_seen": 1163329536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270411233701103, + "loss": 2.5949, + "theoretical_loss": 3.597729026336787, + "tokens_seen": 1163395072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032703109327983956, + "loss": 2.7173, + "theoretical_loss": 3.597710275693731, + "tokens_seen": 1163460608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270210631895687, + "loss": 2.8881, + "theoretical_loss": 3.5976915264025577, + "tokens_seen": 1163526144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270110330992979, + "loss": 2.7011, + "theoretical_loss": 3.5976727784630924, + "tokens_seen": 1163591680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032700100300902705, + "loss": 2.7247, + "theoretical_loss": 3.5976540318751615, + "tokens_seen": 1163657216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269909729187563, + "loss": 2.6947, + "theoretical_loss": 3.597635286638592, + "tokens_seen": 1163722752 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032698094282848547, + "loss": 2.7205, + "theoretical_loss": 3.59761654275321, + "tokens_seen": 1163788288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032697091273821465, + "loss": 2.4699, + "theoretical_loss": 3.597597800218842, + "tokens_seen": 1163853824 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032696088264794383, + "loss": 2.5679, + "theoretical_loss": 3.597579059035314, + "tokens_seen": 1163919360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032695085255767306, + "loss": 2.6229, + "theoretical_loss": 3.5975603192024543, + "tokens_seen": 1163984896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269408224674022, + "loss": 2.8093, + "theoretical_loss": 3.5975415807200886, + "tokens_seen": 1164050432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269307923771314, + "loss": 2.6794, + "theoretical_loss": 3.5975228435880426, + "tokens_seen": 1164115968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032692076228686055, + "loss": 2.6717, + "theoretical_loss": 3.5975041078061443, + "tokens_seen": 1164181504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269107321965898, + "loss": 2.6432, + "theoretical_loss": 3.5974853733742194, + "tokens_seen": 1164247040 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032690070210631897, + "loss": 2.4695, + "theoretical_loss": 3.5974666402920956, + "tokens_seen": 1164312576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032689067201604815, + "loss": 2.6219, + "theoretical_loss": 3.5974479085596, + "tokens_seen": 1164378112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032688064192577733, + "loss": 2.7599, + "theoretical_loss": 3.5974291781765575, + "tokens_seen": 1164443648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268706118355065, + "loss": 2.849, + "theoretical_loss": 3.5974104491427963, + "tokens_seen": 1164509184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268605817452357, + "loss": 2.7382, + "theoretical_loss": 3.5973917214581435, + "tokens_seen": 1164574720 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032685055165496493, + "loss": 2.6985, + "theoretical_loss": 3.5973729951224254, + "tokens_seen": 1164640256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032684052156469406, + "loss": 2.619, + "theoretical_loss": 3.597354270135469, + "tokens_seen": 1164705792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268304914744233, + "loss": 2.4724, + "theoretical_loss": 3.5973355464971015, + "tokens_seen": 1164771328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268204613841524, + "loss": 2.5872, + "theoretical_loss": 3.59731682420715, + "tokens_seen": 1164836864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 647021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5365681648254395, + "objective/train/theoretical_loss": 3.597298103265441, + "objective/train/tokens_used": 1185362400, + "theoretical_loss": 3.597298103265441, + "tokens_seen": 1164902400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032681043129388165, + "loss": 2.7167, + "theoretical_loss": 3.597298103265441, + "tokens_seen": 1164902400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032680040120361084, + "loss": 2.7747, + "theoretical_loss": 3.597279383671802, + "tokens_seen": 1164967936 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032679037111334, + "loss": 2.7435, + "theoretical_loss": 3.5972606654260604, + "tokens_seen": 1165033472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267803410230692, + "loss": 2.5863, + "theoretical_loss": 3.5972419485280422, + "tokens_seen": 1165099008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032677031093279843, + "loss": 2.6398, + "theoretical_loss": 3.597223232977576, + "tokens_seen": 1165164544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032676028084252756, + "loss": 2.77, + "theoretical_loss": 3.597204518774488, + "tokens_seen": 1165230080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267502507522568, + "loss": 2.7466, + "theoretical_loss": 3.5971858059186057, + "tokens_seen": 1165295616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267402206619859, + "loss": 2.5286, + "theoretical_loss": 3.5971670944097562, + "tokens_seen": 1165361152 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032673019057171516, + "loss": 2.6559, + "theoretical_loss": 3.5971483842477676, + "tokens_seen": 1165426688 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032672016048144434, + "loss": 2.7506, + "theoretical_loss": 3.597129675432466, + "tokens_seen": 1165492224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267101303911735, + "loss": 2.6339, + "theoretical_loss": 3.5971109679636797, + "tokens_seen": 1165557760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267001003009027, + "loss": 2.6564, + "theoretical_loss": 3.5970922618412353, + "tokens_seen": 1165623296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266900702106319, + "loss": 2.6649, + "theoretical_loss": 3.5970735570649612, + "tokens_seen": 1165688832 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032668004012036106, + "loss": 2.5365, + "theoretical_loss": 3.597054853634684, + "tokens_seen": 1165754368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266700100300903, + "loss": 2.5501, + "theoretical_loss": 3.597036151550231, + "tokens_seen": 1165819904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266599799398194, + "loss": 2.8168, + "theoretical_loss": 3.597017450811431, + "tokens_seen": 1165885440 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032664994984954866, + "loss": 2.511, + "theoretical_loss": 3.5969987514181105, + "tokens_seen": 1165950976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032663991975927784, + "loss": 2.8327, + "theoretical_loss": 3.596980053370097, + "tokens_seen": 1166016512 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326629889669007, + "loss": 2.77, + "theoretical_loss": 3.5969613566672187, + "tokens_seen": 1166082048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266198595787362, + "loss": 2.8883, + "theoretical_loss": 3.596942661309303, + "tokens_seen": 1166147584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266098294884654, + "loss": 2.9158, + "theoretical_loss": 3.5969239672961777, + "tokens_seen": 1166213120 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032659979939819457, + "loss": 2.8605, + "theoretical_loss": 3.5969052746276704, + "tokens_seen": 1166278656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265897693079238, + "loss": 2.8674, + "theoretical_loss": 3.596886583303608, + "tokens_seen": 1166344192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032657973921765293, + "loss": 2.4217, + "theoretical_loss": 3.59686789332382, + "tokens_seen": 1166409728 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032656970912738216, + "loss": 2.7101, + "theoretical_loss": 3.596849204688133, + "tokens_seen": 1166475264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 647459, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1023616790771484, + "objective/train/theoretical_loss": 3.5968305173963744, + "objective/train/tokens_used": 1187000800, + "theoretical_loss": 3.5968305173963744, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032655967903711135, + "loss": 2.8593, + "theoretical_loss": 3.5968305173963744, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265496489468405, + "loss": 2.5972, + "theoretical_loss": 3.596811831448373, + "tokens_seen": 1166606336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032653961885656976, + "loss": 2.7343, + "theoretical_loss": 3.596793146843957, + "tokens_seen": 1166671872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265295887662989, + "loss": 2.5819, + "theoretical_loss": 3.5967744635829533, + "tokens_seen": 1166737408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265195586760281, + "loss": 2.665, + "theoretical_loss": 3.5967557816651903, + "tokens_seen": 1166802944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032650952858575725, + "loss": 2.6375, + "theoretical_loss": 3.596737101090496, + "tokens_seen": 1166868480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264994984954865, + "loss": 2.5396, + "theoretical_loss": 3.5967184218586983, + "tokens_seen": 1166934016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032648946840521567, + "loss": 2.6808, + "theoretical_loss": 3.5966997439696256, + "tokens_seen": 1166999552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032647943831494485, + "loss": 2.6276, + "theoretical_loss": 3.596681067423106, + "tokens_seen": 1167065088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032646940822467403, + "loss": 2.5045, + "theoretical_loss": 3.596662392218967, + "tokens_seen": 1167130624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032645937813440326, + "loss": 2.4189, + "theoretical_loss": 3.5966437183570372, + "tokens_seen": 1167196160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264493480441324, + "loss": 2.7956, + "theoretical_loss": 3.596625045837145, + "tokens_seen": 1167261696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264393179538616, + "loss": 2.559, + "theoretical_loss": 3.5966063746591184, + "tokens_seen": 1167327232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032642928786359075, + "loss": 2.7304, + "theoretical_loss": 3.596587704822785, + "tokens_seen": 1167392768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032641925777332, + "loss": 2.6686, + "theoretical_loss": 3.596569036327974, + "tokens_seen": 1167458304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032640922768304917, + "loss": 2.476, + "theoretical_loss": 3.596550369174513, + "tokens_seen": 1167523840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032639919759277835, + "loss": 2.5304, + "theoretical_loss": 3.596531703362231, + "tokens_seen": 1167589376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032638916750250753, + "loss": 2.7702, + "theoretical_loss": 3.596513038890956, + "tokens_seen": 1167654912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263791374122367, + "loss": 2.6531, + "theoretical_loss": 3.596494375760516, + "tokens_seen": 1167720448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263691073219659, + "loss": 2.7071, + "theoretical_loss": 3.59647571397074, + "tokens_seen": 1167785984 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032635907723169513, + "loss": 2.8008, + "theoretical_loss": 3.596457053521457, + "tokens_seen": 1167851520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032634904714142426, + "loss": 2.6172, + "theoretical_loss": 3.5964383944124947, + "tokens_seen": 1167917056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263390170511535, + "loss": 2.8027, + "theoretical_loss": 3.596419736643681, + "tokens_seen": 1167982592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263289869608826, + "loss": 2.5849, + "theoretical_loss": 3.5964010802148456, + "tokens_seen": 1168048128 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032631895687061185, + "loss": 2.8076, + "theoretical_loss": 3.596382425125817, + "tokens_seen": 1168113664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 648684, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9439427852630615, + "objective/train/theoretical_loss": 3.5963637713764234, + "objective/train/tokens_used": 1188639200, + "theoretical_loss": 3.5963637713764234, + "tokens_seen": 1168179200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032630892678034104, + "loss": 2.6827, + "theoretical_loss": 3.5963637713764234, + "tokens_seen": 1168179200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262988966900702, + "loss": 2.7814, + "theoretical_loss": 3.5963451189664934, + "tokens_seen": 1168244736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262888665997994, + "loss": 2.6662, + "theoretical_loss": 3.596326467895856, + "tokens_seen": 1168310272 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032627883650952863, + "loss": 2.5774, + "theoretical_loss": 3.5963078181643398, + "tokens_seen": 1168375808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032626880641925776, + "loss": 2.7241, + "theoretical_loss": 3.5962891697717736, + "tokens_seen": 1168441344 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326258776328987, + "loss": 2.6469, + "theoretical_loss": 3.596270522717986, + "tokens_seen": 1168506880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262487462387161, + "loss": 2.5015, + "theoretical_loss": 3.596251877002806, + "tokens_seen": 1168572416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032623871614844536, + "loss": 2.8421, + "theoretical_loss": 3.596233232626062, + "tokens_seen": 1168637952 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032622868605817454, + "loss": 2.7734, + "theoretical_loss": 3.596214589587584, + "tokens_seen": 1168703488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262186559679037, + "loss": 2.7135, + "theoretical_loss": 3.5961959478871997, + "tokens_seen": 1168769024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262086258776329, + "loss": 2.5319, + "theoretical_loss": 3.596177307524738, + "tokens_seen": 1168834560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261985957873621, + "loss": 2.6577, + "theoretical_loss": 3.5961586685000295, + "tokens_seen": 1168900096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032618856569709126, + "loss": 2.4889, + "theoretical_loss": 3.596140030812901, + "tokens_seen": 1168965632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261785356068205, + "loss": 2.5488, + "theoretical_loss": 3.596121394463183, + "tokens_seen": 1169031168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261685055165496, + "loss": 2.6289, + "theoretical_loss": 3.5961027594507042, + "tokens_seen": 1169096704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032615847542627886, + "loss": 2.5204, + "theoretical_loss": 3.5960841257752936, + "tokens_seen": 1169162240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032614844533600804, + "loss": 2.8722, + "theoretical_loss": 3.5960654934367806, + "tokens_seen": 1169227776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261384152457372, + "loss": 2.8192, + "theoretical_loss": 3.5960468624349935, + "tokens_seen": 1169293312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261283851554664, + "loss": 2.7604, + "theoretical_loss": 3.596028232769762, + "tokens_seen": 1169358848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261183550651956, + "loss": 2.5521, + "theoretical_loss": 3.5960096044409164, + "tokens_seen": 1169424384 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032610832497492477, + "loss": 2.8361, + "theoretical_loss": 3.595990977448284, + "tokens_seen": 1169489920 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326098294884654, + "loss": 2.7227, + "theoretical_loss": 3.595972351791696, + "tokens_seen": 1169555456 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032608826479438313, + "loss": 2.698, + "theoretical_loss": 3.59595372747098, + "tokens_seen": 1169620992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032607823470411236, + "loss": 2.6011, + "theoretical_loss": 3.5959351044859664, + "tokens_seen": 1169686528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260682046138415, + "loss": 2.7583, + "theoretical_loss": 3.5959164828364845, + "tokens_seen": 1169752064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 649144, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.792980670928955, + "objective/train/theoretical_loss": 3.5958978625223628, + "objective/train/tokens_used": 1190277600, + "theoretical_loss": 3.5958978625223628, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260581745235707, + "loss": 2.5967, + "theoretical_loss": 3.5958978625223628, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260481444332999, + "loss": 2.7829, + "theoretical_loss": 3.595879243543432, + "tokens_seen": 1169883136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260381143430291, + "loss": 2.6743, + "theoretical_loss": 3.5958606258995207, + "tokens_seen": 1169948672 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032602808425275827, + "loss": 2.5841, + "theoretical_loss": 3.5958420095904593, + "tokens_seen": 1170014208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032601805416248745, + "loss": 2.8274, + "theoretical_loss": 3.5958233946160765, + "tokens_seen": 1170079744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032600802407221663, + "loss": 2.6443, + "theoretical_loss": 3.595804780976202, + "tokens_seen": 1170145280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032599799398194587, + "loss": 2.3263, + "theoretical_loss": 3.5957861686706654, + "tokens_seen": 1170210816 + }, + { + "epoch": 0.35, + "learning_rate": 0.000325987963891675, + "loss": 2.4542, + "theoretical_loss": 3.595767557699297, + "tokens_seen": 1170276352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032597793380140423, + "loss": 2.7791, + "theoretical_loss": 3.595748948061925, + "tokens_seen": 1170341888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259679037111334, + "loss": 2.6827, + "theoretical_loss": 3.59573033975838, + "tokens_seen": 1170407424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259578736208626, + "loss": 2.6128, + "theoretical_loss": 3.5957117327884927, + "tokens_seen": 1170472960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259478435305918, + "loss": 2.8103, + "theoretical_loss": 3.595693127152091, + "tokens_seen": 1170538496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032593781344032095, + "loss": 2.7226, + "theoretical_loss": 3.5956745228490057, + "tokens_seen": 1170604032 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032592778335005014, + "loss": 2.8914, + "theoretical_loss": 3.5956559198790665, + "tokens_seen": 1170669568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032591775325977937, + "loss": 2.6374, + "theoretical_loss": 3.595637318242103, + "tokens_seen": 1170735104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259077231695085, + "loss": 2.682, + "theoretical_loss": 3.5956187179379455, + "tokens_seen": 1170800640 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032589769307923773, + "loss": 2.6491, + "theoretical_loss": 3.595600118966424, + "tokens_seen": 1170866176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032588766298896686, + "loss": 2.7347, + "theoretical_loss": 3.5955815213273676, + "tokens_seen": 1170931712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258776328986961, + "loss": 2.5143, + "theoretical_loss": 3.595562925020607, + "tokens_seen": 1170997248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258676028084253, + "loss": 2.7273, + "theoretical_loss": 3.595544330045972, + "tokens_seen": 1171062784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032585757271815446, + "loss": 2.7069, + "theoretical_loss": 3.5955257364032924, + "tokens_seen": 1171128320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032584754262788364, + "loss": 2.4612, + "theoretical_loss": 3.595507144092399, + "tokens_seen": 1171193856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258375125376128, + "loss": 2.8743, + "theoretical_loss": 3.595488553113121, + "tokens_seen": 1171259392 + }, + { + "epoch": 0.35, + "learning_rate": 0.000325827482447342, + "loss": 2.6314, + "theoretical_loss": 3.5954699634652894, + "tokens_seen": 1171324928 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032581745235707124, + "loss": 2.4695, + "theoretical_loss": 3.595451375148733, + "tokens_seen": 1171390464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 649993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.923766851425171, + "objective/train/theoretical_loss": 3.5954327881632837, + "objective/train/tokens_used": 1191916000, + "theoretical_loss": 3.5954327881632837, + "tokens_seen": 1171456000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258074222668004, + "loss": 2.8658, + "theoretical_loss": 3.5954327881632837, + "tokens_seen": 1171456000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257973921765296, + "loss": 2.7553, + "theoretical_loss": 3.5954142025087705, + "tokens_seen": 1171521536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032578736208625883, + "loss": 2.7047, + "theoretical_loss": 3.595395618185024, + "tokens_seen": 1171587072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032577733199598796, + "loss": 3.073, + "theoretical_loss": 3.5953770351918743, + "tokens_seen": 1171652608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257673019057172, + "loss": 2.5345, + "theoretical_loss": 3.595358453529152, + "tokens_seen": 1171718144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257572718154463, + "loss": 2.5694, + "theoretical_loss": 3.595339873196688, + "tokens_seen": 1171783680 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032574724172517556, + "loss": 2.8471, + "theoretical_loss": 3.5953212941943113, + "tokens_seen": 1171849216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032573721163490474, + "loss": 2.6878, + "theoretical_loss": 3.5953027165218536, + "tokens_seen": 1171914752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257271815446339, + "loss": 2.6934, + "theoretical_loss": 3.595284140179144, + "tokens_seen": 1171980288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257171514543631, + "loss": 2.6298, + "theoretical_loss": 3.595265565166015, + "tokens_seen": 1172045824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257071213640923, + "loss": 2.5254, + "theoretical_loss": 3.595246991482295, + "tokens_seen": 1172111360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032569709127382146, + "loss": 2.6913, + "theoretical_loss": 3.5952284191278157, + "tokens_seen": 1172176896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256870611835507, + "loss": 2.5984, + "theoretical_loss": 3.595209848102407, + "tokens_seen": 1172242432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256770310932798, + "loss": 2.5922, + "theoretical_loss": 3.5951912784059, + "tokens_seen": 1172307968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032566700100300906, + "loss": 2.7977, + "theoretical_loss": 3.5951727100381254, + "tokens_seen": 1172373504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032565697091273824, + "loss": 2.6315, + "theoretical_loss": 3.5951541429989136, + "tokens_seen": 1172439040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256469408224674, + "loss": 3.0059, + "theoretical_loss": 3.595135577288095, + "tokens_seen": 1172504576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256369107321966, + "loss": 2.5167, + "theoretical_loss": 3.595117012905501, + "tokens_seen": 1172570112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256268806419258, + "loss": 2.4955, + "theoretical_loss": 3.595098449850962, + "tokens_seen": 1172635648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032561685055165497, + "loss": 2.638, + "theoretical_loss": 3.5950798881243085, + "tokens_seen": 1172701184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256068204613842, + "loss": 2.4716, + "theoretical_loss": 3.5950613277253716, + "tokens_seen": 1172766720 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032559679037111333, + "loss": 2.5471, + "theoretical_loss": 3.595042768653982, + "tokens_seen": 1172832256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032558676028084256, + "loss": 2.829, + "theoretical_loss": 3.595024210909971, + "tokens_seen": 1172897792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255767301905717, + "loss": 2.5036, + "theoretical_loss": 3.5950056544931694, + "tokens_seen": 1172963328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255667001003009, + "loss": 2.784, + "theoretical_loss": 3.594987099403407, + "tokens_seen": 1173028864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 651258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.56889009475708, + "objective/train/theoretical_loss": 3.5949685456405165, + "objective/train/tokens_used": 1193554400, + "theoretical_loss": 3.5949685456405165, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255566700100301, + "loss": 2.6219, + "theoretical_loss": 3.5949685456405165, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255466399197593, + "loss": 2.7571, + "theoretical_loss": 3.594949993204328, + "tokens_seen": 1173159936 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032553660982948847, + "loss": 2.609, + "theoretical_loss": 3.5949314420946727, + "tokens_seen": 1173225472 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032552657973921765, + "loss": 2.7424, + "theoretical_loss": 3.594912892311381, + "tokens_seen": 1173291008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032551654964894683, + "loss": 2.6436, + "theoretical_loss": 3.594894343854285, + "tokens_seen": 1173356544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032550651955867607, + "loss": 2.7312, + "theoretical_loss": 3.5948757967232154, + "tokens_seen": 1173422080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254964894684052, + "loss": 2.657, + "theoretical_loss": 3.594857250918003, + "tokens_seen": 1173487616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032548645937813443, + "loss": 2.8031, + "theoretical_loss": 3.5948387064384795, + "tokens_seen": 1173553152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254764292878636, + "loss": 2.6133, + "theoretical_loss": 3.594820163284476, + "tokens_seen": 1173618688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254663991975928, + "loss": 2.9284, + "theoretical_loss": 3.594801621455824, + "tokens_seen": 1173684224 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325456369107322, + "loss": 2.6838, + "theoretical_loss": 3.594783080952354, + "tokens_seen": 1173749760 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032544633901705115, + "loss": 2.5229, + "theoretical_loss": 3.5947645417738983, + "tokens_seen": 1173815296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032543630892678034, + "loss": 2.6614, + "theoretical_loss": 3.594746003920287, + "tokens_seen": 1173880832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032542627883650957, + "loss": 2.7473, + "theoretical_loss": 3.5947274673913525, + "tokens_seen": 1173946368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254162487462387, + "loss": 2.4982, + "theoretical_loss": 3.594708932186926, + "tokens_seen": 1174011904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032540621865596793, + "loss": 2.7672, + "theoretical_loss": 3.5946903983068386, + "tokens_seen": 1174077440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032539618856569706, + "loss": 2.6318, + "theoretical_loss": 3.594671865750922, + "tokens_seen": 1174142976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253861584754263, + "loss": 2.7851, + "theoretical_loss": 3.5946533345190073, + "tokens_seen": 1174208512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253761283851555, + "loss": 2.595, + "theoretical_loss": 3.5946348046109273, + "tokens_seen": 1174274048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032536609829488466, + "loss": 2.7437, + "theoretical_loss": 3.5946162760265117, + "tokens_seen": 1174339584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032535606820461384, + "loss": 2.9372, + "theoretical_loss": 3.5945977487655933, + "tokens_seen": 1174405120 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325346038114343, + "loss": 2.704, + "theoretical_loss": 3.5945792228280036, + "tokens_seen": 1174470656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253360080240722, + "loss": 2.664, + "theoretical_loss": 3.5945606982135736, + "tokens_seen": 1174536192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032532597793380144, + "loss": 2.7617, + "theoretical_loss": 3.594542174922136, + "tokens_seen": 1174601728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032531594784353056, + "loss": 2.5863, + "theoretical_loss": 3.5945236529535216, + "tokens_seen": 1174667264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 652492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.737172842025757, + "objective/train/theoretical_loss": 3.5945051323075625, + "objective/train/tokens_used": 1195192800, + "theoretical_loss": 3.5945051323075625, + "tokens_seen": 1174732800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253059177532598, + "loss": 2.6773, + "theoretical_loss": 3.5945051323075625, + "tokens_seen": 1174732800 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325295887662989, + "loss": 2.7729, + "theoretical_loss": 3.594486612984091, + "tokens_seen": 1174798336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032528585757271816, + "loss": 2.7024, + "theoretical_loss": 3.5944680949829375, + "tokens_seen": 1174863872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032527582748244734, + "loss": 2.5224, + "theoretical_loss": 3.5944495783039354, + "tokens_seen": 1174929408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252657973921765, + "loss": 2.5866, + "theoretical_loss": 3.5944310629469154, + "tokens_seen": 1174994944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252557673019057, + "loss": 2.7831, + "theoretical_loss": 3.59441254891171, + "tokens_seen": 1175060480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032524573721163494, + "loss": 2.738, + "theoretical_loss": 3.5943940361981506, + "tokens_seen": 1175126016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032523570712136407, + "loss": 2.5638, + "theoretical_loss": 3.59437552480607, + "tokens_seen": 1175191552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252256770310933, + "loss": 2.6825, + "theoretical_loss": 3.5943570147352997, + "tokens_seen": 1175257088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032521564694082243, + "loss": 2.6556, + "theoretical_loss": 3.594338505985671, + "tokens_seen": 1175322624 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032520561685055166, + "loss": 2.8164, + "theoretical_loss": 3.594319998557017, + "tokens_seen": 1175388160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032519558676028084, + "loss": 2.7041, + "theoretical_loss": 3.59430149244917, + "tokens_seen": 1175453696 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032518555667001, + "loss": 2.5987, + "theoretical_loss": 3.5942829876619613, + "tokens_seen": 1175519232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251755265797392, + "loss": 2.5567, + "theoretical_loss": 3.594264484195223, + "tokens_seen": 1175584768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032516549648946844, + "loss": 2.7074, + "theoretical_loss": 3.594245982048788, + "tokens_seen": 1175650304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032515546639919757, + "loss": 2.7701, + "theoretical_loss": 3.5942274812224877, + "tokens_seen": 1175715840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251454363089268, + "loss": 2.6326, + "theoretical_loss": 3.5942089817161547, + "tokens_seen": 1175781376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032513540621865593, + "loss": 2.608, + "theoretical_loss": 3.594190483529621, + "tokens_seen": 1175846912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032512537612838517, + "loss": 2.5178, + "theoretical_loss": 3.5941719866627193, + "tokens_seen": 1175912448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032511534603811435, + "loss": 2.737, + "theoretical_loss": 3.594153491115282, + "tokens_seen": 1175977984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032510531594784353, + "loss": 2.5786, + "theoretical_loss": 3.5941349968871408, + "tokens_seen": 1176043520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250952858575727, + "loss": 2.5329, + "theoretical_loss": 3.5941165039781287, + "tokens_seen": 1176109056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250852557673019, + "loss": 2.4487, + "theoretical_loss": 3.594098012388078, + "tokens_seen": 1176174592 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032507522567703107, + "loss": 2.5077, + "theoretical_loss": 3.5940795221168207, + "tokens_seen": 1176240128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250651955867603, + "loss": 2.5841, + "theoretical_loss": 3.59406103316419, + "tokens_seen": 1176305664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 652965, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.588792085647583, + "objective/train/theoretical_loss": 3.5940425455300176, + "objective/train/tokens_used": 1196831200, + "theoretical_loss": 3.5940425455300176, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250551654964895, + "loss": 2.6309, + "theoretical_loss": 3.5940425455300176, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032504513540621867, + "loss": 2.7376, + "theoretical_loss": 3.594024059214137, + "tokens_seen": 1176436736 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032503510531594785, + "loss": 2.9167, + "theoretical_loss": 3.59400557421638, + "tokens_seen": 1176502272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032502507522567703, + "loss": 2.6344, + "theoretical_loss": 3.5939870905365794, + "tokens_seen": 1176567808 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032501504513540627, + "loss": 2.6758, + "theoretical_loss": 3.593968608174568, + "tokens_seen": 1176633344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250050150451354, + "loss": 2.6595, + "theoretical_loss": 3.593950127130178, + "tokens_seen": 1176698880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032499498495486463, + "loss": 2.6477, + "theoretical_loss": 3.5939316474032426, + "tokens_seen": 1176764416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249849548645938, + "loss": 2.6468, + "theoretical_loss": 3.593913168993594, + "tokens_seen": 1176829952 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324974924774323, + "loss": 2.7797, + "theoretical_loss": 3.5938946919010655, + "tokens_seen": 1176895488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249648946840522, + "loss": 2.6107, + "theoretical_loss": 3.59387621612549, + "tokens_seen": 1176961024 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032495486459378135, + "loss": 2.6228, + "theoretical_loss": 3.593857741666699, + "tokens_seen": 1177026560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032494483450351054, + "loss": 2.5313, + "theoretical_loss": 3.593839268524527, + "tokens_seen": 1177092096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032493480441323977, + "loss": 2.5962, + "theoretical_loss": 3.593820796698806, + "tokens_seen": 1177157632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249247743229689, + "loss": 2.7831, + "theoretical_loss": 3.593802326189369, + "tokens_seen": 1177223168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032491474423269813, + "loss": 2.6414, + "theoretical_loss": 3.5937838569960494, + "tokens_seen": 1177288704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032490471414242726, + "loss": 2.8084, + "theoretical_loss": 3.5937653891186794, + "tokens_seen": 1177354240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248946840521565, + "loss": 2.7652, + "theoretical_loss": 3.5937469225570924, + "tokens_seen": 1177419776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248846539618857, + "loss": 2.8432, + "theoretical_loss": 3.593728457311122, + "tokens_seen": 1177485312 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032487462387161486, + "loss": 2.6533, + "theoretical_loss": 3.5937099933805996, + "tokens_seen": 1177550848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032486459378134404, + "loss": 2.6939, + "theoretical_loss": 3.59369153076536, + "tokens_seen": 1177616384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248545636910732, + "loss": 2.4783, + "theoretical_loss": 3.5936730694652352, + "tokens_seen": 1177681920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248445336008024, + "loss": 2.8857, + "theoretical_loss": 3.593654609480059, + "tokens_seen": 1177747456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032483450351053164, + "loss": 2.8108, + "theoretical_loss": 3.5936361508096644, + "tokens_seen": 1177812992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032482447342026076, + "loss": 2.7793, + "theoretical_loss": 3.5936176934538846, + "tokens_seen": 1177878528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032481444332999, + "loss": 2.6711, + "theoretical_loss": 3.593599237412553, + "tokens_seen": 1177944064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 652965, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3456740379333496, + "objective/train/theoretical_loss": 3.5935807826855024, + "objective/train/tokens_used": 1198469600, + "theoretical_loss": 3.5935807826855024, + "tokens_seen": 1178009600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248044132397192, + "loss": 2.476, + "theoretical_loss": 3.5935807826855024, + "tokens_seen": 1178009600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032479438314944836, + "loss": 2.7488, + "theoretical_loss": 3.593562329272567, + "tokens_seen": 1178075136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032478435305917754, + "loss": 2.6266, + "theoretical_loss": 3.5935438771735786, + "tokens_seen": 1178140672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247743229689067, + "loss": 2.5096, + "theoretical_loss": 3.593525426388372, + "tokens_seen": 1178206208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247642928786359, + "loss": 2.8226, + "theoretical_loss": 3.59350697691678, + "tokens_seen": 1178271744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032475426278836514, + "loss": 2.5758, + "theoretical_loss": 3.593488528758636, + "tokens_seen": 1178337280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032474423269809427, + "loss": 2.7812, + "theoretical_loss": 3.5934700819137735, + "tokens_seen": 1178402816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247342026078235, + "loss": 2.5558, + "theoretical_loss": 3.593451636382026, + "tokens_seen": 1178468352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032472417251755263, + "loss": 2.8785, + "theoretical_loss": 3.5934331921632268, + "tokens_seen": 1178533888 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032471414242728186, + "loss": 2.5582, + "theoretical_loss": 3.59341474925721, + "tokens_seen": 1178599424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032470411233701104, + "loss": 2.4771, + "theoretical_loss": 3.593396307663809, + "tokens_seen": 1178664960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246940822467402, + "loss": 2.7665, + "theoretical_loss": 3.593377867382857, + "tokens_seen": 1178730496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246840521564694, + "loss": 2.6074, + "theoretical_loss": 3.593359428414188, + "tokens_seen": 1178796032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032467402206619864, + "loss": 2.727, + "theoretical_loss": 3.5933409907576355, + "tokens_seen": 1178861568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032466399197592777, + "loss": 2.8406, + "theoretical_loss": 3.593322554413033, + "tokens_seen": 1178927104 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324653961885657, + "loss": 2.5648, + "theoretical_loss": 3.593304119380215, + "tokens_seen": 1178992640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032464393179538613, + "loss": 2.9887, + "theoretical_loss": 3.593285685659014, + "tokens_seen": 1179058176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032463390170511537, + "loss": 2.558, + "theoretical_loss": 3.5932672532492647, + "tokens_seen": 1179123712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032462387161484455, + "loss": 2.4303, + "theoretical_loss": 3.5932488221508008, + "tokens_seen": 1179189248 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032461384152457373, + "loss": 2.68, + "theoretical_loss": 3.593230392363456, + "tokens_seen": 1179254784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246038114343029, + "loss": 2.7311, + "theoretical_loss": 3.5932119638870645, + "tokens_seen": 1179320320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245937813440321, + "loss": 2.4551, + "theoretical_loss": 3.5931935367214596, + "tokens_seen": 1179385856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032458375125376127, + "loss": 2.7065, + "theoretical_loss": 3.5931751108664756, + "tokens_seen": 1179451392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245737211634905, + "loss": 2.8443, + "theoretical_loss": 3.5931566863219464, + "tokens_seen": 1179516928 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032456369107321963, + "loss": 2.7742, + "theoretical_loss": 3.5931382630877065, + "tokens_seen": 1179582464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 654223, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4708504676818848, + "objective/train/theoretical_loss": 3.593119841163589, + "objective/train/tokens_used": 1200108000, + "theoretical_loss": 3.593119841163589, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032455366098294887, + "loss": 2.6758, + "theoretical_loss": 3.593119841163589, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324543630892678, + "loss": 3.057, + "theoretical_loss": 3.593101420549429, + "tokens_seen": 1179713536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032453360080240723, + "loss": 2.6789, + "theoretical_loss": 3.593083001245059, + "tokens_seen": 1179779072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245235707121364, + "loss": 2.8285, + "theoretical_loss": 3.5930645832503147, + "tokens_seen": 1179844608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245135406218656, + "loss": 2.6795, + "theoretical_loss": 3.5930461665650295, + "tokens_seen": 1179910144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245035105315948, + "loss": 2.6497, + "theoretical_loss": 3.5930277511890383, + "tokens_seen": 1179975680 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324493480441324, + "loss": 2.676, + "theoretical_loss": 3.593009337122174, + "tokens_seen": 1180041216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032448345035105314, + "loss": 2.731, + "theoretical_loss": 3.5929909243642726, + "tokens_seen": 1180106752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244734202607824, + "loss": 2.6065, + "theoretical_loss": 3.592972512915167, + "tokens_seen": 1180172288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244633901705115, + "loss": 2.7483, + "theoretical_loss": 3.592954102774691, + "tokens_seen": 1180237824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032445336008024074, + "loss": 2.4523, + "theoretical_loss": 3.5929356939426804, + "tokens_seen": 1180303360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244433299899699, + "loss": 2.4251, + "theoretical_loss": 3.592917286418969, + "tokens_seen": 1180368896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244332998996991, + "loss": 2.6842, + "theoretical_loss": 3.592898880203391, + "tokens_seen": 1180434432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244232698094283, + "loss": 2.6927, + "theoretical_loss": 3.592880475295781, + "tokens_seen": 1180499968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032441323971915746, + "loss": 2.7892, + "theoretical_loss": 3.5928620716959734, + "tokens_seen": 1180565504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032440320962888664, + "loss": 2.6003, + "theoretical_loss": 3.592843669403803, + "tokens_seen": 1180631040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243931795386159, + "loss": 2.5782, + "theoretical_loss": 3.5928252684191033, + "tokens_seen": 1180696576 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324383149448345, + "loss": 2.6193, + "theoretical_loss": 3.59280686874171, + "tokens_seen": 1180762112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032437311935807424, + "loss": 2.6897, + "theoretical_loss": 3.592788470371457, + "tokens_seen": 1180827648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032436308926780337, + "loss": 2.7288, + "theoretical_loss": 3.5927700733081798, + "tokens_seen": 1180893184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243530591775326, + "loss": 2.6043, + "theoretical_loss": 3.5927516775517114, + "tokens_seen": 1180958720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243430290872618, + "loss": 2.7239, + "theoretical_loss": 3.5927332831018877, + "tokens_seen": 1181024256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032433299899699096, + "loss": 2.5689, + "theoretical_loss": 3.5927148899585433, + "tokens_seen": 1181089792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243229689067202, + "loss": 2.8611, + "theoretical_loss": 3.5926964981215126, + "tokens_seen": 1181155328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243129388164494, + "loss": 2.531, + "theoretical_loss": 3.5926781075906304, + "tokens_seen": 1181220864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 655414, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6210334300994873, + "objective/train/theoretical_loss": 3.5926597183657316, + "objective/train/tokens_used": 1201746400, + "theoretical_loss": 3.5926597183657316, + "tokens_seen": 1181286400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032430290872617856, + "loss": 2.7083, + "theoretical_loss": 3.5926597183657316, + "tokens_seen": 1181286400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032429287863590774, + "loss": 2.6754, + "theoretical_loss": 3.592641330446651, + "tokens_seen": 1181351936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242828485456369, + "loss": 2.4938, + "theoretical_loss": 3.592622943833223, + "tokens_seen": 1181417472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242728184553661, + "loss": 2.5009, + "theoretical_loss": 3.5926045585252835, + "tokens_seen": 1181483008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032426278836509534, + "loss": 2.7818, + "theoretical_loss": 3.5925861745226664, + "tokens_seen": 1181548544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032425275827482447, + "loss": 2.7641, + "theoretical_loss": 3.592567791825207, + "tokens_seen": 1181614080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242427281845537, + "loss": 2.7128, + "theoretical_loss": 3.5925494104327402, + "tokens_seen": 1181679616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032423269809428283, + "loss": 2.8377, + "theoretical_loss": 3.5925310303451012, + "tokens_seen": 1181745152 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032422266800401206, + "loss": 2.6549, + "theoretical_loss": 3.592512651562125, + "tokens_seen": 1181810688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032421263791374125, + "loss": 2.502, + "theoretical_loss": 3.592494274083646, + "tokens_seen": 1181876224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242026078234704, + "loss": 2.6759, + "theoretical_loss": 3.5924758979095004, + "tokens_seen": 1181941760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241925777331996, + "loss": 2.6429, + "theoretical_loss": 3.5924575230395224, + "tokens_seen": 1182007296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032418254764292884, + "loss": 2.3383, + "theoretical_loss": 3.592439149473547, + "tokens_seen": 1182072832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032417251755265797, + "loss": 2.6148, + "theoretical_loss": 3.5924207772114105, + "tokens_seen": 1182138368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241624874623872, + "loss": 2.6313, + "theoretical_loss": 3.5924024062529476, + "tokens_seen": 1182203904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032415245737211633, + "loss": 2.7198, + "theoretical_loss": 3.5923840365979927, + "tokens_seen": 1182269440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032414242728184557, + "loss": 2.6309, + "theoretical_loss": 3.592365668246382, + "tokens_seen": 1182334976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032413239719157475, + "loss": 2.937, + "theoretical_loss": 3.5923473011979508, + "tokens_seen": 1182400512 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032412236710130393, + "loss": 2.6589, + "theoretical_loss": 3.5923289354525334, + "tokens_seen": 1182466048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241123370110331, + "loss": 2.8447, + "theoretical_loss": 3.5923105710099663, + "tokens_seen": 1182531584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241023069207623, + "loss": 2.7621, + "theoretical_loss": 3.5922922078700847, + "tokens_seen": 1182597120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032409227683049147, + "loss": 2.7971, + "theoretical_loss": 3.5922738460327235, + "tokens_seen": 1182662656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240822467402207, + "loss": 2.8789, + "theoretical_loss": 3.5922554854977182, + "tokens_seen": 1182728192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032407221664994984, + "loss": 2.6273, + "theoretical_loss": 3.592237126264905, + "tokens_seen": 1182793728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032406218655967907, + "loss": 2.8434, + "theoretical_loss": 3.5922187683341185, + "tokens_seen": 1182859264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 655986, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3905091285705566, + "objective/train/theoretical_loss": 3.5922004117051944, + "objective/train/tokens_used": 1203384800, + "theoretical_loss": 3.5922004117051944, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240521564694082, + "loss": 2.5661, + "theoretical_loss": 3.5922004117051944, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032404212637913743, + "loss": 2.5477, + "theoretical_loss": 3.592182056377969, + "tokens_seen": 1182990336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240320962888666, + "loss": 2.7005, + "theoretical_loss": 3.592163702352277, + "tokens_seen": 1183055872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240220661985958, + "loss": 2.7133, + "theoretical_loss": 3.5921453496279545, + "tokens_seen": 1183121408 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324012036108325, + "loss": 2.7503, + "theoretical_loss": 3.5921269982048374, + "tokens_seen": 1183186944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240020060180542, + "loss": 2.6778, + "theoretical_loss": 3.5921086480827604, + "tokens_seen": 1183252480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032399197592778334, + "loss": 2.7109, + "theoretical_loss": 3.59209029926156, + "tokens_seen": 1183318016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239819458375126, + "loss": 2.545, + "theoretical_loss": 3.592071951741072, + "tokens_seen": 1183383552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239719157472417, + "loss": 2.7016, + "theoretical_loss": 3.592053605521132, + "tokens_seen": 1183449088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032396188565697094, + "loss": 2.8656, + "theoretical_loss": 3.592035260601575, + "tokens_seen": 1183514624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239518555667001, + "loss": 2.4547, + "theoretical_loss": 3.5920169169822382, + "tokens_seen": 1183580160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239418254764293, + "loss": 2.7549, + "theoretical_loss": 3.5919985746629566, + "tokens_seen": 1183645696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239317953861585, + "loss": 2.8236, + "theoretical_loss": 3.5919802336435662, + "tokens_seen": 1183711232 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032392176529588766, + "loss": 2.4944, + "theoretical_loss": 3.5919618939239033, + "tokens_seen": 1183776768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032391173520561684, + "loss": 2.7984, + "theoretical_loss": 3.5919435555038035, + "tokens_seen": 1183842304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239017051153461, + "loss": 2.6072, + "theoretical_loss": 3.5919252183831025, + "tokens_seen": 1183907840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238916750250752, + "loss": 2.6405, + "theoretical_loss": 3.591906882561637, + "tokens_seen": 1183973376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032388164493480444, + "loss": 2.9206, + "theoretical_loss": 3.5918885480392424, + "tokens_seen": 1184038912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032387161484453357, + "loss": 2.7311, + "theoretical_loss": 3.5918702148157555, + "tokens_seen": 1184104448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238615847542628, + "loss": 2.7311, + "theoretical_loss": 3.591851882891012, + "tokens_seen": 1184169984 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323851554663992, + "loss": 2.7366, + "theoretical_loss": 3.5918335522648475, + "tokens_seen": 1184235520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032384152457372116, + "loss": 2.3489, + "theoretical_loss": 3.5918152229370985, + "tokens_seen": 1184301056 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032383149448345034, + "loss": 2.6593, + "theoretical_loss": 3.591796894907602, + "tokens_seen": 1184366592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238214643931796, + "loss": 2.8426, + "theoretical_loss": 3.591778568176193, + "tokens_seen": 1184432128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238114343029087, + "loss": 2.7458, + "theoretical_loss": 3.5917602427427084, + "tokens_seen": 1184497664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 656832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.695746421813965, + "objective/train/theoretical_loss": 3.591741918606984, + "objective/train/tokens_used": 1205023200, + "theoretical_loss": 3.591741918606984, + "tokens_seen": 1184563200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032380140421263794, + "loss": 2.874, + "theoretical_loss": 3.591741918606984, + "tokens_seen": 1184563200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032379137412236707, + "loss": 2.6529, + "theoretical_loss": 3.5917235957688574, + "tokens_seen": 1184628736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237813440320963, + "loss": 2.6853, + "theoretical_loss": 3.591705274228163, + "tokens_seen": 1184694272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237713139418255, + "loss": 2.8111, + "theoretical_loss": 3.591686953984739, + "tokens_seen": 1184759808 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032376128385155467, + "loss": 2.6361, + "theoretical_loss": 3.5916686350384204, + "tokens_seen": 1184825344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032375125376128385, + "loss": 2.5834, + "theoretical_loss": 3.5916503173890444, + "tokens_seen": 1184890880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032374122367101303, + "loss": 2.905, + "theoretical_loss": 3.591632001036447, + "tokens_seen": 1184956416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237311935807422, + "loss": 2.6953, + "theoretical_loss": 3.591613685980465, + "tokens_seen": 1185021952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032372116349047145, + "loss": 2.8276, + "theoretical_loss": 3.5915953722209353, + "tokens_seen": 1185087488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032371113340020057, + "loss": 2.5766, + "theoretical_loss": 3.5915770597576935, + "tokens_seen": 1185153024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237011033099298, + "loss": 2.586, + "theoretical_loss": 3.591558748590577, + "tokens_seen": 1185218560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032369107321965893, + "loss": 2.7271, + "theoretical_loss": 3.591540438719422, + "tokens_seen": 1185284096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032368104312938817, + "loss": 2.8175, + "theoretical_loss": 3.5915221301440647, + "tokens_seen": 1185349632 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032367101303911735, + "loss": 2.823, + "theoretical_loss": 3.5915038228643423, + "tokens_seen": 1185415168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032366098294884653, + "loss": 2.7177, + "theoretical_loss": 3.5914855168800917, + "tokens_seen": 1185480704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236509528585757, + "loss": 2.648, + "theoretical_loss": 3.5914672121911493, + "tokens_seen": 1185546240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032364092276830495, + "loss": 2.8443, + "theoretical_loss": 3.591448908797352, + "tokens_seen": 1185611776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236308926780341, + "loss": 2.7258, + "theoretical_loss": 3.591430606698536, + "tokens_seen": 1185677312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236208625877633, + "loss": 2.5077, + "theoretical_loss": 3.591412305894539, + "tokens_seen": 1185742848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032361083249749244, + "loss": 2.7522, + "theoretical_loss": 3.5913940063851975, + "tokens_seen": 1185808384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236008024072217, + "loss": 2.5417, + "theoretical_loss": 3.591375708170348, + "tokens_seen": 1185873920 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032359077231695085, + "loss": 2.8327, + "theoretical_loss": 3.5913574112498274, + "tokens_seen": 1185939456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032358074222668004, + "loss": 2.6164, + "theoretical_loss": 3.591339115623474, + "tokens_seen": 1186004992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032357071213640927, + "loss": 2.4979, + "theoretical_loss": 3.5913208212911227, + "tokens_seen": 1186070528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235606820461384, + "loss": 2.7355, + "theoretical_loss": 3.591302528252612, + "tokens_seen": 1186136064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 657301, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4484574794769287, + "objective/train/theoretical_loss": 3.5912842365077777, + "objective/train/tokens_used": 1206661600, + "theoretical_loss": 3.5912842365077777, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032355065195586763, + "loss": 2.6418, + "theoretical_loss": 3.5912842365077777, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235406218655968, + "loss": 2.7045, + "theoretical_loss": 3.591265946056458, + "tokens_seen": 1186267136 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323530591775326, + "loss": 2.6399, + "theoretical_loss": 3.5912476568984895, + "tokens_seen": 1186332672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235205616850552, + "loss": 2.7264, + "theoretical_loss": 3.5912293690337096, + "tokens_seen": 1186398208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235105315947844, + "loss": 2.6608, + "theoretical_loss": 3.5912110824619545, + "tokens_seen": 1186463744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032350050150451354, + "loss": 2.6546, + "theoretical_loss": 3.591192797183062, + "tokens_seen": 1186529280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234904714142428, + "loss": 2.6396, + "theoretical_loss": 3.59117451319687, + "tokens_seen": 1186594816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234804413239719, + "loss": 2.711, + "theoretical_loss": 3.5911562305032145, + "tokens_seen": 1186660352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032347041123370114, + "loss": 2.735, + "theoretical_loss": 3.5911379491019333, + "tokens_seen": 1186725888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234603811434303, + "loss": 2.7617, + "theoretical_loss": 3.5911196689928637, + "tokens_seen": 1186791424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234503510531595, + "loss": 2.6371, + "theoretical_loss": 3.591101390175843, + "tokens_seen": 1186856960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234403209628887, + "loss": 2.6863, + "theoretical_loss": 3.5910831126507086, + "tokens_seen": 1186922496 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032343029087261786, + "loss": 2.7775, + "theoretical_loss": 3.5910648364172975, + "tokens_seen": 1186988032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032342026078234704, + "loss": 2.9409, + "theoretical_loss": 3.591046561475447, + "tokens_seen": 1187053568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234102306920763, + "loss": 2.9129, + "theoretical_loss": 3.5910282878249955, + "tokens_seen": 1187119104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234002006018054, + "loss": 2.7658, + "theoretical_loss": 3.5910100154657796, + "tokens_seen": 1187184640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032339017051153464, + "loss": 2.5633, + "theoretical_loss": 3.590991744397637, + "tokens_seen": 1187250176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032338014042126377, + "loss": 2.6758, + "theoretical_loss": 3.5909734746204056, + "tokens_seen": 1187315712 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323370110330993, + "loss": 2.6059, + "theoretical_loss": 3.5909552061339225, + "tokens_seen": 1187381248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233600802407222, + "loss": 2.6783, + "theoretical_loss": 3.5909369389380252, + "tokens_seen": 1187446784 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032335005015045136, + "loss": 2.5929, + "theoretical_loss": 3.5909186730325517, + "tokens_seen": 1187512320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032334002006018054, + "loss": 2.6622, + "theoretical_loss": 3.590900408417339, + "tokens_seen": 1187577856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233299899699098, + "loss": 2.4147, + "theoretical_loss": 3.5908821450922255, + "tokens_seen": 1187643392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233199598796389, + "loss": 2.5664, + "theoretical_loss": 3.5908638830570485, + "tokens_seen": 1187708928 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032330992978936814, + "loss": 2.6783, + "theoretical_loss": 3.5908456223116456, + "tokens_seen": 1187774464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 658636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8977458477020264, + "objective/train/theoretical_loss": 3.590827362855855, + "objective/train/tokens_used": 1208300000, + "theoretical_loss": 3.590827362855855, + "tokens_seen": 1187840000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032329989969909727, + "loss": 2.8993, + "theoretical_loss": 3.590827362855855, + "tokens_seen": 1187840000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232898696088265, + "loss": 2.895, + "theoretical_loss": 3.590809104689514, + "tokens_seen": 1187905536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232798395185557, + "loss": 2.6605, + "theoretical_loss": 3.5907908478124613, + "tokens_seen": 1187971072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032326980942828487, + "loss": 2.7268, + "theoretical_loss": 3.5907725922245337, + "tokens_seen": 1188036608 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032325977933801405, + "loss": 2.4671, + "theoretical_loss": 3.5907543379255693, + "tokens_seen": 1188102144 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032324974924774323, + "loss": 2.5579, + "theoretical_loss": 3.590736084915406, + "tokens_seen": 1188167680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232397191574724, + "loss": 2.5451, + "theoretical_loss": 3.5907178331938825, + "tokens_seen": 1188233216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032322968906720165, + "loss": 2.8114, + "theoretical_loss": 3.5906995827608355, + "tokens_seen": 1188298752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032321965897693077, + "loss": 2.7601, + "theoretical_loss": 3.5906813336161036, + "tokens_seen": 1188364288 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032320962888666, + "loss": 2.8106, + "theoretical_loss": 3.590663085759525, + "tokens_seen": 1188429824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032319959879638913, + "loss": 2.6096, + "theoretical_loss": 3.5906448391909382, + "tokens_seen": 1188495360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032318956870611837, + "loss": 2.7287, + "theoretical_loss": 3.59062659391018, + "tokens_seen": 1188560896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032317953861584755, + "loss": 2.734, + "theoretical_loss": 3.5906083499170895, + "tokens_seen": 1188626432 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032316950852557673, + "loss": 2.9007, + "theoretical_loss": 3.590590107211504, + "tokens_seen": 1188691968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231594784353059, + "loss": 2.624, + "theoretical_loss": 3.5905718657932626, + "tokens_seen": 1188757504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032314944834503515, + "loss": 2.5711, + "theoretical_loss": 3.590553625662203, + "tokens_seen": 1188823040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231394182547643, + "loss": 2.6533, + "theoretical_loss": 3.5905353868181633, + "tokens_seen": 1188888576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231293881644935, + "loss": 2.6393, + "theoretical_loss": 3.5905171492609815, + "tokens_seen": 1188954112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032311935807422264, + "loss": 2.7133, + "theoretical_loss": 3.5904989129904967, + "tokens_seen": 1189019648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231093279839519, + "loss": 2.5837, + "theoretical_loss": 3.5904806780065472, + "tokens_seen": 1189085184 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032309929789368105, + "loss": 2.8506, + "theoretical_loss": 3.5904624443089705, + "tokens_seen": 1189150720 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032308926780341024, + "loss": 2.9039, + "theoretical_loss": 3.590444211897606, + "tokens_seen": 1189216256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230792377131394, + "loss": 2.7202, + "theoretical_loss": 3.5904259807722907, + "tokens_seen": 1189281792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230692076228686, + "loss": 2.6139, + "theoretical_loss": 3.5904077509328642, + "tokens_seen": 1189347328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230591775325978, + "loss": 2.5795, + "theoretical_loss": 3.590389522379165, + "tokens_seen": 1189412864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 659771, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9299919605255127, + "objective/train/theoretical_loss": 3.5903712951110305, + "objective/train/tokens_used": 1209938400, + "theoretical_loss": 3.5903712951110305, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323049147442327, + "loss": 2.7602, + "theoretical_loss": 3.5903712951110305, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032303911735205614, + "loss": 2.4382, + "theoretical_loss": 3.5903530691283, + "tokens_seen": 1189543936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230290872617854, + "loss": 2.8542, + "theoretical_loss": 3.590334844430812, + "tokens_seen": 1189609472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230190571715145, + "loss": 2.6869, + "theoretical_loss": 3.590316621018405, + "tokens_seen": 1189675008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032300902708124374, + "loss": 2.7318, + "theoretical_loss": 3.5902983988909183, + "tokens_seen": 1189740544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229989969909729, + "loss": 2.6897, + "theoretical_loss": 3.5902801780481886, + "tokens_seen": 1189806080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229889669007021, + "loss": 2.5943, + "theoretical_loss": 3.590261958490057, + "tokens_seen": 1189871616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229789368104313, + "loss": 2.5884, + "theoretical_loss": 3.5902437402163603, + "tokens_seen": 1189937152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229689067201605, + "loss": 2.7583, + "theoretical_loss": 3.5902255232269376, + "tokens_seen": 1190002688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032295887662988964, + "loss": 2.6598, + "theoretical_loss": 3.5902073075216285, + "tokens_seen": 1190068224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229488465396189, + "loss": 2.6565, + "theoretical_loss": 3.590189093100271, + "tokens_seen": 1190133760 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322938816449348, + "loss": 2.7323, + "theoretical_loss": 3.590170879962704, + "tokens_seen": 1190199296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032292878635907724, + "loss": 2.5401, + "theoretical_loss": 3.590152668108767, + "tokens_seen": 1190264832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229187562688064, + "loss": 2.7671, + "theoretical_loss": 3.590134457538298, + "tokens_seen": 1190330368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229087261785356, + "loss": 2.6562, + "theoretical_loss": 3.590116248251136, + "tokens_seen": 1190395904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228986960882648, + "loss": 2.5468, + "theoretical_loss": 3.59009804024712, + "tokens_seen": 1190461440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032288866599799397, + "loss": 2.8533, + "theoretical_loss": 3.5900798335260897, + "tokens_seen": 1190526976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032287863590772315, + "loss": 2.8279, + "theoretical_loss": 3.590061628087883, + "tokens_seen": 1190592512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228686058174524, + "loss": 2.6782, + "theoretical_loss": 3.5900434239323396, + "tokens_seen": 1190658048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228585757271815, + "loss": 2.668, + "theoretical_loss": 3.590025221059298, + "tokens_seen": 1190723584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032284854563691074, + "loss": 2.6663, + "theoretical_loss": 3.5900070194685982, + "tokens_seen": 1190789120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228385155466399, + "loss": 2.9798, + "theoretical_loss": 3.589988819160078, + "tokens_seen": 1190854656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228284854563691, + "loss": 2.5456, + "theoretical_loss": 3.5899706201335775, + "tokens_seen": 1190920192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032281845536609834, + "loss": 2.7142, + "theoretical_loss": 3.5899524223889356, + "tokens_seen": 1190985728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032280842527582747, + "loss": 2.6882, + "theoretical_loss": 3.589934225925991, + "tokens_seen": 1191051264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 660497, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.826659917831421, + "objective/train/theoretical_loss": 3.589916030744584, + "objective/train/tokens_used": 1211576800, + "theoretical_loss": 3.589916030744584, + "tokens_seen": 1191116800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227983951855567, + "loss": 2.8398, + "theoretical_loss": 3.589916030744584, + "tokens_seen": 1191116800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227883650952859, + "loss": 2.536, + "theoretical_loss": 3.589897836844553, + "tokens_seen": 1191182336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032277833500501507, + "loss": 2.5995, + "theoretical_loss": 3.589879644225737, + "tokens_seen": 1191247872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032276830491474425, + "loss": 2.7609, + "theoretical_loss": 3.589861452887976, + "tokens_seen": 1191313408 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032275827482447343, + "loss": 2.5637, + "theoretical_loss": 3.5898432628311094, + "tokens_seen": 1191378944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227482447342026, + "loss": 2.8383, + "theoretical_loss": 3.5898250740549758, + "tokens_seen": 1191444480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032273821464393185, + "loss": 2.5828, + "theoretical_loss": 3.5898068865594155, + "tokens_seen": 1191510016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032272818455366097, + "loss": 2.7367, + "theoretical_loss": 3.589788700344267, + "tokens_seen": 1191575552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227181544633902, + "loss": 2.7134, + "theoretical_loss": 3.5897705154093704, + "tokens_seen": 1191641088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032270812437311933, + "loss": 2.535, + "theoretical_loss": 3.589752331754565, + "tokens_seen": 1191706624 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032269809428284857, + "loss": 2.6662, + "theoretical_loss": 3.58973414937969, + "tokens_seen": 1191772160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032268806419257775, + "loss": 2.5705, + "theoretical_loss": 3.589715968284585, + "tokens_seen": 1191837696 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032267803410230693, + "loss": 2.7252, + "theoretical_loss": 3.58969778846909, + "tokens_seen": 1191903232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226680040120361, + "loss": 2.596, + "theoretical_loss": 3.589679609933045, + "tokens_seen": 1191968768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032265797392176535, + "loss": 2.5493, + "theoretical_loss": 3.589661432676288, + "tokens_seen": 1192034304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226479438314945, + "loss": 2.5605, + "theoretical_loss": 3.58964325669866, + "tokens_seen": 1192099840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226379137412237, + "loss": 2.7074, + "theoretical_loss": 3.589625082, + "tokens_seen": 1192165376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032262788365095284, + "loss": 2.7322, + "theoretical_loss": 3.5896069085801483, + "tokens_seen": 1192230912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226178535606821, + "loss": 2.8322, + "theoretical_loss": 3.5895887364389445, + "tokens_seen": 1192296448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032260782347041125, + "loss": 2.6919, + "theoretical_loss": 3.589570565576228, + "tokens_seen": 1192361984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032259779338014044, + "loss": 2.708, + "theoretical_loss": 3.5895523959918383, + "tokens_seen": 1192427520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225877632898696, + "loss": 2.7512, + "theoretical_loss": 3.5895342276856157, + "tokens_seen": 1192493056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225777331995988, + "loss": 2.5763, + "theoretical_loss": 3.5895160606574006, + "tokens_seen": 1192558592 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322567703109328, + "loss": 2.7169, + "theoretical_loss": 3.5894978949070317, + "tokens_seen": 1192624128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225576730190572, + "loss": 2.4533, + "theoretical_loss": 3.58947973043435, + "tokens_seen": 1192689664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 661795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.610649347305298, + "objective/train/theoretical_loss": 3.5894615672391947, + "objective/train/tokens_used": 1213215200, + "theoretical_loss": 3.5894615672391947, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032254764292878634, + "loss": 2.6943, + "theoretical_loss": 3.5894615672391947, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225376128385156, + "loss": 2.7775, + "theoretical_loss": 3.589443405321406, + "tokens_seen": 1192820736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225275827482447, + "loss": 2.541, + "theoretical_loss": 3.5894252446808244, + "tokens_seen": 1192886272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032251755265797394, + "loss": 2.6342, + "theoretical_loss": 3.589407085317289, + "tokens_seen": 1192951808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225075225677031, + "loss": 2.2844, + "theoretical_loss": 3.5893889272306403, + "tokens_seen": 1193017344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224974924774323, + "loss": 2.7089, + "theoretical_loss": 3.589370770420718, + "tokens_seen": 1193082880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224874623871615, + "loss": 2.4681, + "theoretical_loss": 3.589352614887363, + "tokens_seen": 1193148416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224774322968907, + "loss": 2.833, + "theoretical_loss": 3.5893344606304147, + "tokens_seen": 1193213952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032246740220661984, + "loss": 2.5555, + "theoretical_loss": 3.589316307649714, + "tokens_seen": 1193279488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224573721163491, + "loss": 2.656, + "theoretical_loss": 3.5892981559451003, + "tokens_seen": 1193345024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224473420260782, + "loss": 2.5065, + "theoretical_loss": 3.5892800055164145, + "tokens_seen": 1193410560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032243731193580744, + "loss": 2.4987, + "theoretical_loss": 3.589261856363496, + "tokens_seen": 1193476096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224272818455366, + "loss": 2.7401, + "theoretical_loss": 3.589243708486186, + "tokens_seen": 1193541632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224172517552658, + "loss": 2.5925, + "theoretical_loss": 3.5892255618843247, + "tokens_seen": 1193607168 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322407221664995, + "loss": 2.8549, + "theoretical_loss": 3.589207416557752, + "tokens_seen": 1193672704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032239719157472417, + "loss": 2.7572, + "theoretical_loss": 3.589189272506308, + "tokens_seen": 1193738240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032238716148445335, + "loss": 2.676, + "theoretical_loss": 3.5891711297298334, + "tokens_seen": 1193803776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223771313941826, + "loss": 2.5692, + "theoretical_loss": 3.5891529882281694, + "tokens_seen": 1193869312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223671013039117, + "loss": 2.6654, + "theoretical_loss": 3.5891348480011556, + "tokens_seen": 1193934848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032235707121364095, + "loss": 2.4853, + "theoretical_loss": 3.589116709048633, + "tokens_seen": 1194000384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223470411233701, + "loss": 2.5927, + "theoretical_loss": 3.589098571370441, + "tokens_seen": 1194065920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223370110330993, + "loss": 2.6482, + "theoretical_loss": 3.589080434966421, + "tokens_seen": 1194131456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223269809428285, + "loss": 2.5284, + "theoretical_loss": 3.5890622998364146, + "tokens_seen": 1194196992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032231695085255767, + "loss": 2.65, + "theoretical_loss": 3.58904416598026, + "tokens_seen": 1194262528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032230692076228685, + "loss": 2.4034, + "theoretical_loss": 3.5890260333977997, + "tokens_seen": 1194328064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 662071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3905045986175537, + "objective/train/theoretical_loss": 3.589007902088874, + "objective/train/tokens_used": 1214853600, + "theoretical_loss": 3.589007902088874, + "tokens_seen": 1194393600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222968906720161, + "loss": 2.4546, + "theoretical_loss": 3.589007902088874, + "tokens_seen": 1194393600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222868605817452, + "loss": 2.8831, + "theoretical_loss": 3.5889897720533233, + "tokens_seen": 1194459136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032227683049147445, + "loss": 2.6318, + "theoretical_loss": 3.5889716432909875, + "tokens_seen": 1194524672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222668004012036, + "loss": 2.4259, + "theoretical_loss": 3.588953515801709, + "tokens_seen": 1194590208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222567703109328, + "loss": 2.7315, + "theoretical_loss": 3.5889353895853278, + "tokens_seen": 1194655744 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322246740220662, + "loss": 2.7919, + "theoretical_loss": 3.588917264641685, + "tokens_seen": 1194721280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032223671013039117, + "loss": 2.5828, + "theoretical_loss": 3.5888991409706206, + "tokens_seen": 1194786816 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032222668004012035, + "loss": 2.6261, + "theoretical_loss": 3.5888810185719757, + "tokens_seen": 1194852352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032221664994984953, + "loss": 2.4561, + "theoretical_loss": 3.5888628974455914, + "tokens_seen": 1194917888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222066198595787, + "loss": 2.5928, + "theoretical_loss": 3.5888447775913095, + "tokens_seen": 1194983424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032219658976930795, + "loss": 2.5253, + "theoretical_loss": 3.5888266590089692, + "tokens_seen": 1195048960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221865596790371, + "loss": 2.6944, + "theoretical_loss": 3.588808541698413, + "tokens_seen": 1195114496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221765295887663, + "loss": 2.4546, + "theoretical_loss": 3.588790425659481, + "tokens_seen": 1195180032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221664994984955, + "loss": 2.6356, + "theoretical_loss": 3.588772310892015, + "tokens_seen": 1195245568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221564694082247, + "loss": 2.6211, + "theoretical_loss": 3.5887541973958546, + "tokens_seen": 1195311104 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032214643931795386, + "loss": 2.758, + "theoretical_loss": 3.5887360851708423, + "tokens_seen": 1195376640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032213640922768304, + "loss": 2.7939, + "theoretical_loss": 3.5887179742168187, + "tokens_seen": 1195442176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221263791374122, + "loss": 2.616, + "theoretical_loss": 3.588699864533625, + "tokens_seen": 1195507712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032211634904714145, + "loss": 2.6652, + "theoretical_loss": 3.5886817561211024, + "tokens_seen": 1195573248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221063189568706, + "loss": 2.7012, + "theoretical_loss": 3.588663648979092, + "tokens_seen": 1195638784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220962888665998, + "loss": 2.7739, + "theoretical_loss": 3.588645543107435, + "tokens_seen": 1195704320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032208625877632894, + "loss": 2.8342, + "theoretical_loss": 3.5886274385059727, + "tokens_seen": 1195769856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220762286860582, + "loss": 2.6931, + "theoretical_loss": 3.5886093351745467, + "tokens_seen": 1195835392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220661985957874, + "loss": 2.5972, + "theoretical_loss": 3.5885912331129974, + "tokens_seen": 1195900928 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032205616850551654, + "loss": 2.6297, + "theoretical_loss": 3.5885731323211676, + "tokens_seen": 1195966464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 663277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6097090244293213, + "objective/train/theoretical_loss": 3.5885550327988973, + "objective/train/tokens_used": 1216492000, + "theoretical_loss": 3.5885550327988973, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220461384152458, + "loss": 2.7212, + "theoretical_loss": 3.5885550327988973, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220361083249749, + "loss": 2.7139, + "theoretical_loss": 3.5885369345460285, + "tokens_seen": 1196097536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032202607823470414, + "loss": 2.5494, + "theoretical_loss": 3.5885188375624026, + "tokens_seen": 1196163072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220160481444333, + "loss": 2.6198, + "theoretical_loss": 3.588500741847861, + "tokens_seen": 1196228608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220060180541625, + "loss": 2.9059, + "theoretical_loss": 3.5884826474022455, + "tokens_seen": 1196294144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219959879638917, + "loss": 2.7009, + "theoretical_loss": 3.588464554225397, + "tokens_seen": 1196359680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219859578736209, + "loss": 2.7996, + "theoretical_loss": 3.5884464623171572, + "tokens_seen": 1196425216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032197592778335004, + "loss": 2.7252, + "theoretical_loss": 3.588428371677368, + "tokens_seen": 1196490752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219658976930793, + "loss": 2.623, + "theoretical_loss": 3.588410282305871, + "tokens_seen": 1196556288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219558676028084, + "loss": 2.7392, + "theoretical_loss": 3.588392194202507, + "tokens_seen": 1196621824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032194583751253764, + "loss": 2.8584, + "theoretical_loss": 3.5883741073671187, + "tokens_seen": 1196687360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219358074222668, + "loss": 2.6099, + "theoretical_loss": 3.5883560217995476, + "tokens_seen": 1196752896 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321925777331996, + "loss": 2.8514, + "theoretical_loss": 3.588337937499635, + "tokens_seen": 1196818432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219157472417252, + "loss": 2.5845, + "theoretical_loss": 3.5883198544672226, + "tokens_seen": 1196883968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032190571715145437, + "loss": 2.6211, + "theoretical_loss": 3.588301772702153, + "tokens_seen": 1196949504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032189568706118355, + "loss": 2.783, + "theoretical_loss": 3.588283692204267, + "tokens_seen": 1197015040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218856569709128, + "loss": 2.7052, + "theoretical_loss": 3.5882656129734065, + "tokens_seen": 1197080576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218756268806419, + "loss": 2.7137, + "theoretical_loss": 3.5882475350094136, + "tokens_seen": 1197146112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032186559679037115, + "loss": 2.5264, + "theoretical_loss": 3.5882294583121306, + "tokens_seen": 1197211648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218555667001003, + "loss": 2.5417, + "theoretical_loss": 3.5882113828813993, + "tokens_seen": 1197277184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218455366098295, + "loss": 2.6667, + "theoretical_loss": 3.588193308717061, + "tokens_seen": 1197342720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218355065195587, + "loss": 2.6857, + "theoretical_loss": 3.5881752358189583, + "tokens_seen": 1197408256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032182547642928787, + "loss": 2.7712, + "theoretical_loss": 3.588157164186933, + "tokens_seen": 1197473792 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032181544633901705, + "loss": 2.6639, + "theoretical_loss": 3.5881390938208266, + "tokens_seen": 1197539328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218054162487463, + "loss": 2.7148, + "theoretical_loss": 3.588121024720482, + "tokens_seen": 1197604864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 664536, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6411828994750977, + "objective/train/theoretical_loss": 3.588102956885741, + "objective/train/tokens_used": 1218130400, + "theoretical_loss": 3.588102956885741, + "tokens_seen": 1197670400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217953861584754, + "loss": 2.83, + "theoretical_loss": 3.588102956885741, + "tokens_seen": 1197670400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032178535606820465, + "loss": 2.5972, + "theoretical_loss": 3.5880848903164457, + "tokens_seen": 1197735936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217753259779338, + "loss": 2.7223, + "theoretical_loss": 3.5880668250124375, + "tokens_seen": 1197801472 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321765295887663, + "loss": 2.7356, + "theoretical_loss": 3.5880487609735594, + "tokens_seen": 1197867008 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217552657973922, + "loss": 2.7346, + "theoretical_loss": 3.588030698199654, + "tokens_seen": 1197932544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032174523570712137, + "loss": 2.575, + "theoretical_loss": 3.5880126366905625, + "tokens_seen": 1197998080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032173520561685055, + "loss": 2.4085, + "theoretical_loss": 3.5879945764461274, + "tokens_seen": 1198063616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032172517552657974, + "loss": 2.6027, + "theoretical_loss": 3.5879765174661915, + "tokens_seen": 1198129152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217151454363089, + "loss": 2.6716, + "theoretical_loss": 3.5879584597505967, + "tokens_seen": 1198194688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032170511534603815, + "loss": 2.61, + "theoretical_loss": 3.5879404032991853, + "tokens_seen": 1198260224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216950852557673, + "loss": 2.5906, + "theoretical_loss": 3.5879223481117997, + "tokens_seen": 1198325760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216850551654965, + "loss": 2.5357, + "theoretical_loss": 3.5879042941882826, + "tokens_seen": 1198391296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216750250752257, + "loss": 2.8647, + "theoretical_loss": 3.5878862415284765, + "tokens_seen": 1198456832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216649949849549, + "loss": 2.5841, + "theoretical_loss": 3.5878681901322227, + "tokens_seen": 1198522368 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032165496489468406, + "loss": 2.6733, + "theoretical_loss": 3.5878501399993645, + "tokens_seen": 1198587904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032164493480441324, + "loss": 2.5762, + "theoretical_loss": 3.587832091129745, + "tokens_seen": 1198653440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216349047141424, + "loss": 2.6126, + "theoretical_loss": 3.587814043523206, + "tokens_seen": 1198718976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032162487462387165, + "loss": 2.3205, + "theoretical_loss": 3.5877959971795903, + "tokens_seen": 1198784512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216148445336008, + "loss": 2.6872, + "theoretical_loss": 3.58777795209874, + "tokens_seen": 1198850048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032160481444333, + "loss": 2.7524, + "theoretical_loss": 3.587759908280498, + "tokens_seen": 1198915584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032159478435305914, + "loss": 2.7265, + "theoretical_loss": 3.5877418657247073, + "tokens_seen": 1198981120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215847542627884, + "loss": 2.7187, + "theoretical_loss": 3.5877238244312104, + "tokens_seen": 1199046656 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032157472417251756, + "loss": 2.8152, + "theoretical_loss": 3.5877057843998497, + "tokens_seen": 1199112192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032156469408224674, + "loss": 2.6784, + "theoretical_loss": 3.5876877456304683, + "tokens_seen": 1199177728 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215546639919759, + "loss": 2.6942, + "theoretical_loss": 3.5876697081229088, + "tokens_seen": 1199243264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 665055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5700125694274902, + "objective/train/theoretical_loss": 3.587651671877014, + "objective/train/tokens_used": 1219768800, + "theoretical_loss": 3.587651671877014, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215446339017051, + "loss": 2.676, + "theoretical_loss": 3.587651671877014, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215346038114343, + "loss": 2.5606, + "theoretical_loss": 3.587633636892627, + "tokens_seen": 1199374336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215245737211635, + "loss": 2.5459, + "theoretical_loss": 3.5876156031695903, + "tokens_seen": 1199439872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032151454363089265, + "loss": 2.517, + "theoretical_loss": 3.587597570707746, + "tokens_seen": 1199505408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215045135406219, + "loss": 2.6024, + "theoretical_loss": 3.5875795395069385, + "tokens_seen": 1199570944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032149448345035106, + "loss": 2.8159, + "theoretical_loss": 3.58756150956701, + "tokens_seen": 1199636480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032148445336008024, + "loss": 2.6025, + "theoretical_loss": 3.5875434808878035, + "tokens_seen": 1199702016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214744232698094, + "loss": 2.4649, + "theoretical_loss": 3.587525453469162, + "tokens_seen": 1199767552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214643931795386, + "loss": 2.8485, + "theoretical_loss": 3.5875074273109284, + "tokens_seen": 1199833088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214543630892678, + "loss": 2.6042, + "theoretical_loss": 3.587489402412946, + "tokens_seen": 1199898624 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321444332998997, + "loss": 2.7668, + "theoretical_loss": 3.5874713787750574, + "tokens_seen": 1199964160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032143430290872615, + "loss": 2.9209, + "theoretical_loss": 3.587453356397106, + "tokens_seen": 1200029696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214242728184554, + "loss": 2.8785, + "theoretical_loss": 3.587435335278935, + "tokens_seen": 1200095232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214142427281845, + "loss": 2.4521, + "theoretical_loss": 3.587417315420388, + "tokens_seen": 1200160768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032140421263791375, + "loss": 2.8288, + "theoretical_loss": 3.5873992968213066, + "tokens_seen": 1200226304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032139418254764293, + "loss": 2.6307, + "theoretical_loss": 3.5873812794815363, + "tokens_seen": 1200291840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213841524573721, + "loss": 2.8476, + "theoretical_loss": 3.587363263400918, + "tokens_seen": 1200357376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213741223671013, + "loss": 2.7485, + "theoretical_loss": 3.5873452485792967, + "tokens_seen": 1200422912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213640922768305, + "loss": 2.7168, + "theoretical_loss": 3.587327235016515, + "tokens_seen": 1200488448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032135406218655965, + "loss": 2.4365, + "theoretical_loss": 3.587309222712416, + "tokens_seen": 1200553984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213440320962889, + "loss": 2.6422, + "theoretical_loss": 3.5872912116668436, + "tokens_seen": 1200619520 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321334002006018, + "loss": 2.6353, + "theoretical_loss": 3.5872732018796407, + "tokens_seen": 1200685056 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032132397191574725, + "loss": 2.6725, + "theoretical_loss": 3.587255193350651, + "tokens_seen": 1200750592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213139418254765, + "loss": 2.6841, + "theoretical_loss": 3.587237186079718, + "tokens_seen": 1200816128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213039117352056, + "loss": 2.5766, + "theoretical_loss": 3.587219180066685, + "tokens_seen": 1200881664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 666208, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7520852088928223, + "objective/train/theoretical_loss": 3.587201175311395, + "objective/train/tokens_used": 1221407200, + "theoretical_loss": 3.587201175311395, + "tokens_seen": 1200947200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032129388164493485, + "loss": 2.8138, + "theoretical_loss": 3.587201175311395, + "tokens_seen": 1200947200 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321283851554664, + "loss": 2.8434, + "theoretical_loss": 3.5871831718136926, + "tokens_seen": 1201012736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212738214643932, + "loss": 2.8096, + "theoretical_loss": 3.5871651695734204, + "tokens_seen": 1201078272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212637913741224, + "loss": 2.446, + "theoretical_loss": 3.5871471685904224, + "tokens_seen": 1201143808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212537612838516, + "loss": 2.709, + "theoretical_loss": 3.5871291688645424, + "tokens_seen": 1201209344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032124373119358075, + "loss": 2.5374, + "theoretical_loss": 3.5871111703956235, + "tokens_seen": 1201274880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032123370110330994, + "loss": 2.5882, + "theoretical_loss": 3.5870931731835096, + "tokens_seen": 1201340416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212236710130391, + "loss": 2.496, + "theoretical_loss": 3.5870751772280443, + "tokens_seen": 1201405952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032121364092276835, + "loss": 2.4976, + "theoretical_loss": 3.5870571825290716, + "tokens_seen": 1201471488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212036108324975, + "loss": 2.6233, + "theoretical_loss": 3.5870391890864353, + "tokens_seen": 1201537024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211935807422267, + "loss": 2.4834, + "theoretical_loss": 3.587021196899978, + "tokens_seen": 1201602560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211835506519559, + "loss": 2.5302, + "theoretical_loss": 3.5870032059695456, + "tokens_seen": 1201668096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211735205616851, + "loss": 2.6988, + "theoretical_loss": 3.58698521629498, + "tokens_seen": 1201733632 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032116349047141426, + "loss": 2.5982, + "theoretical_loss": 3.586967227876126, + "tokens_seen": 1201799168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032115346038114344, + "loss": 2.7329, + "theoretical_loss": 3.586949240712827, + "tokens_seen": 1201864704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211434302908726, + "loss": 2.6829, + "theoretical_loss": 3.5869312548049277, + "tokens_seen": 1201930240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032113340020060185, + "loss": 2.883, + "theoretical_loss": 3.5869132701522712, + "tokens_seen": 1201995776 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321123370110331, + "loss": 2.6068, + "theoretical_loss": 3.586895286754702, + "tokens_seen": 1202061312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211133400200602, + "loss": 2.6619, + "theoretical_loss": 3.586877304612064, + "tokens_seen": 1202126848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032110330992978934, + "loss": 2.6131, + "theoretical_loss": 3.5868593237242004, + "tokens_seen": 1202192384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210932798395186, + "loss": 2.737, + "theoretical_loss": 3.5868413440909563, + "tokens_seen": 1202257920 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032108324974924776, + "loss": 2.747, + "theoretical_loss": 3.5868233657121755, + "tokens_seen": 1202323456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032107321965897694, + "loss": 2.7392, + "theoretical_loss": 3.5868053885877016, + "tokens_seen": 1202388992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210631895687061, + "loss": 2.5638, + "theoretical_loss": 3.5867874127173796, + "tokens_seen": 1202454528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210531594784353, + "loss": 2.6974, + "theoretical_loss": 3.5867694381010535, + "tokens_seen": 1202520064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 666794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.74890398979187, + "objective/train/theoretical_loss": 3.5867514647385663, + "objective/train/tokens_used": 1223045600, + "theoretical_loss": 3.5867514647385663, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210431293881645, + "loss": 2.7497, + "theoretical_loss": 3.5867514647385663, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210330992978937, + "loss": 2.5142, + "theoretical_loss": 3.586733492629764, + "tokens_seen": 1202651136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032102306920762285, + "loss": 2.6491, + "theoretical_loss": 3.5867155217744893, + "tokens_seen": 1202716672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210130391173521, + "loss": 2.8934, + "theoretical_loss": 3.5866975521725877, + "tokens_seen": 1202782208 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032100300902708126, + "loss": 2.7662, + "theoretical_loss": 3.5866795838239023, + "tokens_seen": 1202847744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032099297893681044, + "loss": 2.738, + "theoretical_loss": 3.5866616167282785, + "tokens_seen": 1202913280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209829488465396, + "loss": 2.6258, + "theoretical_loss": 3.58664365088556, + "tokens_seen": 1202978816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209729187562688, + "loss": 2.6219, + "theoretical_loss": 3.5866256862955916, + "tokens_seen": 1203044352 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320962888665998, + "loss": 2.5657, + "theoretical_loss": 3.5866077229582176, + "tokens_seen": 1203109888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209528585757272, + "loss": 2.6135, + "theoretical_loss": 3.5865897608732817, + "tokens_seen": 1203175424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032094282848545635, + "loss": 2.7004, + "theoretical_loss": 3.5865718000406295, + "tokens_seen": 1203240960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209327983951856, + "loss": 2.7194, + "theoretical_loss": 3.586553840460105, + "tokens_seen": 1203306496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209227683049147, + "loss": 2.6563, + "theoretical_loss": 3.5865358821315527, + "tokens_seen": 1203372032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032091273821464395, + "loss": 2.797, + "theoretical_loss": 3.5865179250548174, + "tokens_seen": 1203437568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032090270812437313, + "loss": 2.6798, + "theoretical_loss": 3.586499969229743, + "tokens_seen": 1203503104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208926780341023, + "loss": 2.8114, + "theoretical_loss": 3.5864820146561747, + "tokens_seen": 1203568640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208826479438315, + "loss": 2.5543, + "theoretical_loss": 3.586464061333957, + "tokens_seen": 1203634176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208726178535607, + "loss": 2.6157, + "theoretical_loss": 3.5864461092629343, + "tokens_seen": 1203699712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032086258776328985, + "loss": 2.8626, + "theoretical_loss": 3.586428158442952, + "tokens_seen": 1203765248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208525576730191, + "loss": 2.3889, + "theoretical_loss": 3.5864102088738536, + "tokens_seen": 1203830784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208425275827482, + "loss": 2.8346, + "theoretical_loss": 3.586392260555485, + "tokens_seen": 1203896320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032083249749247745, + "loss": 2.9258, + "theoretical_loss": 3.5863743134876906, + "tokens_seen": 1203961856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032082246740220663, + "loss": 2.5813, + "theoretical_loss": 3.586356367670315, + "tokens_seen": 1204027392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208124373119358, + "loss": 2.7728, + "theoretical_loss": 3.5863384231032036, + "tokens_seen": 1204092928 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320802407221665, + "loss": 2.6154, + "theoretical_loss": 3.5863204797862003, + "tokens_seen": 1204158464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 667638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.698781728744507, + "objective/train/theoretical_loss": 3.586302537719151, + "objective/train/tokens_used": 1224684000, + "theoretical_loss": 3.586302537719151, + "tokens_seen": 1204224000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207923771313942, + "loss": 2.659, + "theoretical_loss": 3.586302537719151, + "tokens_seen": 1204224000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032078234704112336, + "loss": 2.6391, + "theoretical_loss": 3.5862845969018995, + "tokens_seen": 1204289536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207723169508526, + "loss": 2.5357, + "theoretical_loss": 3.586266657334292, + "tokens_seen": 1204355072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207622868605817, + "loss": 2.5632, + "theoretical_loss": 3.5862487190161723, + "tokens_seen": 1204420608 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032075225677031095, + "loss": 2.7713, + "theoretical_loss": 3.5862307819473864, + "tokens_seen": 1204486144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207422266800401, + "loss": 2.5229, + "theoretical_loss": 3.5862128461277782, + "tokens_seen": 1204551680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207321965897693, + "loss": 2.493, + "theoretical_loss": 3.586194911557194, + "tokens_seen": 1204617216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207221664994985, + "loss": 2.8192, + "theoretical_loss": 3.586176978235478, + "tokens_seen": 1204682752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207121364092277, + "loss": 2.8019, + "theoretical_loss": 3.586159046162476, + "tokens_seen": 1204748288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032070210631895686, + "loss": 2.4404, + "theoretical_loss": 3.586141115338032, + "tokens_seen": 1204813824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206920762286861, + "loss": 2.7097, + "theoretical_loss": 3.586123185761992, + "tokens_seen": 1204879360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206820461384152, + "loss": 2.8557, + "theoretical_loss": 3.586105257434201, + "tokens_seen": 1204944896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032067201604814446, + "loss": 2.8357, + "theoretical_loss": 3.5860873303545047, + "tokens_seen": 1205010432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206619859578736, + "loss": 2.8592, + "theoretical_loss": 3.586069404522747, + "tokens_seen": 1205075968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206519558676028, + "loss": 2.8587, + "theoretical_loss": 3.586051479938775, + "tokens_seen": 1205141504 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320641925777332, + "loss": 2.7109, + "theoretical_loss": 3.5860335566024326, + "tokens_seen": 1205207040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206318956870612, + "loss": 2.5293, + "theoretical_loss": 3.586015634513566, + "tokens_seen": 1205272576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032062186559679036, + "loss": 2.7344, + "theoretical_loss": 3.585997713672019, + "tokens_seen": 1205338112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032061183550651954, + "loss": 2.5098, + "theoretical_loss": 3.5859797940776392, + "tokens_seen": 1205403648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206018054162487, + "loss": 2.7707, + "theoretical_loss": 3.5859618757302707, + "tokens_seen": 1205469184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032059177532597796, + "loss": 2.7907, + "theoretical_loss": 3.585943958629759, + "tokens_seen": 1205534720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205817452357071, + "loss": 2.6121, + "theoretical_loss": 3.5859260427759496, + "tokens_seen": 1205600256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205717151454363, + "loss": 2.8092, + "theoretical_loss": 3.5859081281686884, + "tokens_seen": 1205665792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205616850551655, + "loss": 2.4816, + "theoretical_loss": 3.5858902148078204, + "tokens_seen": 1205731328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205516549648947, + "loss": 2.4486, + "theoretical_loss": 3.5858723026931916, + "tokens_seen": 1205796864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 668120, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.765895366668701, + "objective/train/theoretical_loss": 3.585854391824647, + "objective/train/tokens_used": 1226322400, + "theoretical_loss": 3.585854391824647, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205416248746239, + "loss": 3.0271, + "theoretical_loss": 3.585854391824647, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032053159478435305, + "loss": 2.6968, + "theoretical_loss": 3.5858364822020325, + "tokens_seen": 1205927936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205215646940823, + "loss": 2.7393, + "theoretical_loss": 3.585818573825194, + "tokens_seen": 1205993472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032051153460381146, + "loss": 2.6489, + "theoretical_loss": 3.585800666693977, + "tokens_seen": 1206059008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032050150451354064, + "loss": 2.6772, + "theoretical_loss": 3.585782760808227, + "tokens_seen": 1206124544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204914744232698, + "loss": 2.5699, + "theoretical_loss": 3.5857648561677893, + "tokens_seen": 1206190080 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320481444332999, + "loss": 2.8226, + "theoretical_loss": 3.585746952772511, + "tokens_seen": 1206255616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204714142427282, + "loss": 2.678, + "theoretical_loss": 3.5857290506222363, + "tokens_seen": 1206321152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204613841524574, + "loss": 2.6826, + "theoretical_loss": 3.585711149716812, + "tokens_seen": 1206386688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032045135406218655, + "loss": 2.725, + "theoretical_loss": 3.5856932500560834, + "tokens_seen": 1206452224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204413239719158, + "loss": 2.7681, + "theoretical_loss": 3.585675351639897, + "tokens_seen": 1206517760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204312938816449, + "loss": 2.7728, + "theoretical_loss": 3.5856574544680977, + "tokens_seen": 1206583296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032042126379137415, + "loss": 2.6691, + "theoretical_loss": 3.5856395585405325, + "tokens_seen": 1206648832 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032041123370110333, + "loss": 2.6117, + "theoretical_loss": 3.5856216638570464, + "tokens_seen": 1206714368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204012036108325, + "loss": 2.7022, + "theoretical_loss": 3.5856037704174857, + "tokens_seen": 1206779904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203911735205617, + "loss": 2.757, + "theoretical_loss": 3.5855858782216963, + "tokens_seen": 1206845440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203811434302909, + "loss": 2.7985, + "theoretical_loss": 3.585567987269525, + "tokens_seen": 1206910976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032037111334002005, + "loss": 2.3973, + "theoretical_loss": 3.5855500975608163, + "tokens_seen": 1206976512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203610832497493, + "loss": 2.6385, + "theoretical_loss": 3.5855322090954176, + "tokens_seen": 1207042048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203510531594784, + "loss": 2.6184, + "theoretical_loss": 3.585514321873174, + "tokens_seen": 1207107584 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032034102306920765, + "loss": 2.8333, + "theoretical_loss": 3.5854964358939325, + "tokens_seen": 1207173120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032033099297893683, + "loss": 2.6203, + "theoretical_loss": 3.5854785511575393, + "tokens_seen": 1207238656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032033099297893683, + "loss": 2.6739, + "theoretical_loss": 3.5854606676638396, + "tokens_seen": 1207304192 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320320962888666, + "loss": 2.7132, + "theoretical_loss": 3.5854427854126802, + "tokens_seen": 1207369728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203109327983952, + "loss": 2.6528, + "theoretical_loss": 3.585424904403908, + "tokens_seen": 1207435264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 669193, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.440397024154663, + "objective/train/theoretical_loss": 3.5854070246373677, + "objective/train/tokens_used": 1227960800, + "theoretical_loss": 3.5854070246373677, + "tokens_seen": 1207500800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203009027081244, + "loss": 2.4402, + "theoretical_loss": 3.5854070246373677, + "tokens_seen": 1207500800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032029087261785356, + "loss": 2.8087, + "theoretical_loss": 3.585389146112907, + "tokens_seen": 1207566336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202808425275828, + "loss": 2.79, + "theoretical_loss": 3.5853712688303707, + "tokens_seen": 1207631872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202708124373119, + "loss": 2.7522, + "theoretical_loss": 3.585353392789607, + "tokens_seen": 1207697408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032026078234704115, + "loss": 2.6718, + "theoretical_loss": 3.585335517990461, + "tokens_seen": 1207762944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202507522567703, + "loss": 2.6665, + "theoretical_loss": 3.58531764443278, + "tokens_seen": 1207828480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202407221664995, + "loss": 2.4913, + "theoretical_loss": 3.585299772116409, + "tokens_seen": 1207894016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202306920762287, + "loss": 2.8258, + "theoretical_loss": 3.5852819010411956, + "tokens_seen": 1207959552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202206619859579, + "loss": 2.7235, + "theoretical_loss": 3.5852640312069863, + "tokens_seen": 1208025088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032021063189568706, + "loss": 2.5067, + "theoretical_loss": 3.585246162613627, + "tokens_seen": 1208090624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202006018054163, + "loss": 2.7812, + "theoretical_loss": 3.585228295260965, + "tokens_seen": 1208156160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201905717151454, + "loss": 2.7972, + "theoretical_loss": 3.5852104291488462, + "tokens_seen": 1208221696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032018054162487466, + "loss": 2.5162, + "theoretical_loss": 3.5851925642771167, + "tokens_seen": 1208287232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201705115346038, + "loss": 2.7096, + "theoretical_loss": 3.5851747006456245, + "tokens_seen": 1208352768 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320160481444333, + "loss": 2.7143, + "theoretical_loss": 3.585156838254216, + "tokens_seen": 1208418304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201504513540622, + "loss": 2.818, + "theoretical_loss": 3.585138977102736, + "tokens_seen": 1208483840 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201404212637914, + "loss": 2.6218, + "theoretical_loss": 3.5851211171910338, + "tokens_seen": 1208549376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032013039117352056, + "loss": 2.395, + "theoretical_loss": 3.5851032585189544, + "tokens_seen": 1208614912 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032012036108324974, + "loss": 2.3369, + "theoretical_loss": 3.585085401086345, + "tokens_seen": 1208680448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201103309929789, + "loss": 2.5753, + "theoretical_loss": 3.585067544893053, + "tokens_seen": 1208745984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032010030090270816, + "loss": 2.8804, + "theoretical_loss": 3.5850496899389244, + "tokens_seen": 1208811520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200902708124373, + "loss": 2.6475, + "theoretical_loss": 3.5850318362238056, + "tokens_seen": 1208877056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200802407221665, + "loss": 2.8044, + "theoretical_loss": 3.5850139837475448, + "tokens_seen": 1208942592 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032007021063189565, + "loss": 2.5214, + "theoretical_loss": 3.584996132509988, + "tokens_seen": 1209008128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200601805416249, + "loss": 2.7903, + "theoretical_loss": 3.5849782825109826, + "tokens_seen": 1209073664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 670409, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.449265480041504, + "objective/train/theoretical_loss": 3.584960433750375, + "objective/train/tokens_used": 1229599200, + "theoretical_loss": 3.584960433750375, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032005015045135407, + "loss": 2.629, + "theoretical_loss": 3.584960433750375, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032004012036108325, + "loss": 2.6543, + "theoretical_loss": 3.5849425862280127, + "tokens_seen": 1209204736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032003009027081243, + "loss": 2.5549, + "theoretical_loss": 3.5849247399437423, + "tokens_seen": 1209270272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032002006018054166, + "loss": 2.7935, + "theoretical_loss": 3.5849068948974114, + "tokens_seen": 1209335808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200100300902708, + "loss": 2.693, + "theoretical_loss": 3.584889051088866, + "tokens_seen": 1209401344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032, + "loss": 2.8575, + "theoretical_loss": 3.5848712085179537, + "tokens_seen": 1209466880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031998996990972915, + "loss": 2.7422, + "theoretical_loss": 3.584853367184522, + "tokens_seen": 1209532416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199799398194584, + "loss": 2.593, + "theoretical_loss": 3.584835527088418, + "tokens_seen": 1209597952 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031996990972918757, + "loss": 2.6048, + "theoretical_loss": 3.584817688229488, + "tokens_seen": 1209663488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031995987963891675, + "loss": 2.7282, + "theoretical_loss": 3.58479985060758, + "tokens_seen": 1209729024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031994984954864593, + "loss": 2.7549, + "theoretical_loss": 3.584782014222541, + "tokens_seen": 1209794560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199398194583751, + "loss": 2.4356, + "theoretical_loss": 3.5847641790742184, + "tokens_seen": 1209860096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199297893681043, + "loss": 2.677, + "theoretical_loss": 3.584746345162459, + "tokens_seen": 1209925632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031991975927783353, + "loss": 2.7683, + "theoretical_loss": 3.5847285124871107, + "tokens_seen": 1209991168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031990972918756266, + "loss": 2.6618, + "theoretical_loss": 3.58471068104802, + "tokens_seen": 1210056704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198996990972919, + "loss": 2.6729, + "theoretical_loss": 3.5846928508450353, + "tokens_seen": 1210122240 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319889669007021, + "loss": 2.5604, + "theoretical_loss": 3.5846750218780032, + "tokens_seen": 1210187776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031987963891675025, + "loss": 2.5945, + "theoretical_loss": 3.584657194146771, + "tokens_seen": 1210253312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031986960882647943, + "loss": 2.5825, + "theoretical_loss": 3.584639367651187, + "tokens_seen": 1210318848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198595787362086, + "loss": 2.6042, + "theoretical_loss": 3.5846215423910976, + "tokens_seen": 1210384384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198495486459378, + "loss": 2.6627, + "theoretical_loss": 3.5846037183663513, + "tokens_seen": 1210449920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031983951855566703, + "loss": 2.6505, + "theoretical_loss": 3.5845858955767946, + "tokens_seen": 1210515456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031982948846539616, + "loss": 2.5319, + "theoretical_loss": 3.5845680740222763, + "tokens_seen": 1210580992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198194583751254, + "loss": 2.6629, + "theoretical_loss": 3.584550253702642, + "tokens_seen": 1210646528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198094282848546, + "loss": 2.6587, + "theoretical_loss": 3.5845324346177416, + "tokens_seen": 1210712064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 670966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8296167850494385, + "objective/train/theoretical_loss": 3.584514616767421, + "objective/train/tokens_used": 1231237600, + "theoretical_loss": 3.584514616767421, + "tokens_seen": 1210777600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031979939819458376, + "loss": 2.6485, + "theoretical_loss": 3.584514616767421, + "tokens_seen": 1210777600 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319789368104313, + "loss": 2.6005, + "theoretical_loss": 3.5844968001515287, + "tokens_seen": 1210843136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197793380140421, + "loss": 2.715, + "theoretical_loss": 3.584478984769912, + "tokens_seen": 1210908672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031976930792377135, + "loss": 2.689, + "theoretical_loss": 3.584461170622419, + "tokens_seen": 1210974208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197592778335005, + "loss": 2.8714, + "theoretical_loss": 3.584443357708897, + "tokens_seen": 1211039744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197492477432297, + "loss": 2.2651, + "theoretical_loss": 3.584425546029194, + "tokens_seen": 1211105280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197392176529589, + "loss": 2.6919, + "theoretical_loss": 3.5844077355831576, + "tokens_seen": 1211170816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197291875626881, + "loss": 2.6406, + "theoretical_loss": 3.584389926370635, + "tokens_seen": 1211236352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031971915747241726, + "loss": 2.7218, + "theoretical_loss": 3.584372118391476, + "tokens_seen": 1211301888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197091273821465, + "loss": 2.6949, + "theoretical_loss": 3.5843543116455265, + "tokens_seen": 1211367424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196990972918756, + "loss": 2.5578, + "theoretical_loss": 3.584336506132635, + "tokens_seen": 1211432960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031968906720160486, + "loss": 2.6447, + "theoretical_loss": 3.5843187018526494, + "tokens_seen": 1211498496 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319679037111334, + "loss": 2.7112, + "theoretical_loss": 3.5843008988054184, + "tokens_seen": 1211564032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196690070210632, + "loss": 2.6566, + "theoretical_loss": 3.584283096990789, + "tokens_seen": 1211629568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196589769307924, + "loss": 2.6208, + "theoretical_loss": 3.5842652964086095, + "tokens_seen": 1211695104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196489468405216, + "loss": 2.5844, + "theoretical_loss": 3.584247497058728, + "tokens_seen": 1211760640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031963891675025076, + "loss": 2.6369, + "theoretical_loss": 3.5842296989409923, + "tokens_seen": 1211826176 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031962888665997994, + "loss": 2.7917, + "theoretical_loss": 3.5842119020552508, + "tokens_seen": 1211891712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196188565697091, + "loss": 2.7858, + "theoretical_loss": 3.584194106401351, + "tokens_seen": 1211957248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031960882647943836, + "loss": 2.6762, + "theoretical_loss": 3.5841763119791423, + "tokens_seen": 1212022784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195987963891675, + "loss": 2.9897, + "theoretical_loss": 3.5841585187884712, + "tokens_seen": 1212088320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195887662988967, + "loss": 2.5801, + "theoretical_loss": 3.5841407268291876, + "tokens_seen": 1212153856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031957873620862585, + "loss": 2.7895, + "theoretical_loss": 3.5841229361011386, + "tokens_seen": 1212219392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195687061183551, + "loss": 2.7946, + "theoretical_loss": 3.5841051466041725, + "tokens_seen": 1212284928 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031955867602808427, + "loss": 2.6184, + "theoretical_loss": 3.5840873583381376, + "tokens_seen": 1212350464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 672128, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6712498664855957, + "objective/train/theoretical_loss": 3.5840695713028827, + "objective/train/tokens_used": 1232876000, + "theoretical_loss": 3.5840695713028827, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031954864593781345, + "loss": 2.5049, + "theoretical_loss": 3.5840695713028827, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031953861584754263, + "loss": 2.7097, + "theoretical_loss": 3.5840517854982554, + "tokens_seen": 1212481536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031952858575727186, + "loss": 2.6616, + "theoretical_loss": 3.584034000924105, + "tokens_seen": 1212547072 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319518555667001, + "loss": 2.4007, + "theoretical_loss": 3.584016217580279, + "tokens_seen": 1212612608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195085255767302, + "loss": 2.7693, + "theoretical_loss": 3.583998435466626, + "tokens_seen": 1212678144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031949849548645935, + "loss": 2.7655, + "theoretical_loss": 3.5839806545829944, + "tokens_seen": 1212743680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194884653961886, + "loss": 2.6329, + "theoretical_loss": 3.583962874929233, + "tokens_seen": 1212809216 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031947843530591777, + "loss": 2.5438, + "theoretical_loss": 3.5839450965051904, + "tokens_seen": 1212874752 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031946840521564695, + "loss": 2.6286, + "theoretical_loss": 3.583927319310714, + "tokens_seen": 1212940288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031945837512537613, + "loss": 2.411, + "theoretical_loss": 3.5839095433456536, + "tokens_seen": 1213005824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194483450351053, + "loss": 2.4915, + "theoretical_loss": 3.583891768609857, + "tokens_seen": 1213071360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194383149448345, + "loss": 2.588, + "theoretical_loss": 3.5838739951031737, + "tokens_seen": 1213136896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031942828485456373, + "loss": 2.5782, + "theoretical_loss": 3.5838562228254514, + "tokens_seen": 1213202432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031941825476429286, + "loss": 2.6396, + "theoretical_loss": 3.5838384517765385, + "tokens_seen": 1213267968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194082246740221, + "loss": 2.6197, + "theoretical_loss": 3.5838206819562846, + "tokens_seen": 1213333504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193981945837512, + "loss": 2.762, + "theoretical_loss": 3.5838029133645377, + "tokens_seen": 1213399040 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031938816449348045, + "loss": 2.6373, + "theoretical_loss": 3.583785146001147, + "tokens_seen": 1213464576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031937813440320964, + "loss": 2.6973, + "theoretical_loss": 3.583767379865961, + "tokens_seen": 1213530112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193681043129388, + "loss": 2.5355, + "theoretical_loss": 3.5837496149588284, + "tokens_seen": 1213595648 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319358074222668, + "loss": 2.7368, + "theoretical_loss": 3.583731851279598, + "tokens_seen": 1213661184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031934804413239723, + "loss": 2.8614, + "theoretical_loss": 3.583714088828119, + "tokens_seen": 1213726720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031933801404212636, + "loss": 2.5393, + "theoretical_loss": 3.5836963276042404, + "tokens_seen": 1213792256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193279839518556, + "loss": 2.7673, + "theoretical_loss": 3.58367856760781, + "tokens_seen": 1213857792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193179538615847, + "loss": 2.7873, + "theoretical_loss": 3.583660808838678, + "tokens_seen": 1213923328 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031930792377131396, + "loss": 2.758, + "theoretical_loss": 3.5836430512966926, + "tokens_seen": 1213988864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 672740, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2003424167633057, + "objective/train/theoretical_loss": 3.5836252949817027, + "objective/train/tokens_used": 1234514400, + "theoretical_loss": 3.5836252949817027, + "tokens_seen": 1214054400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031929789368104314, + "loss": 2.7173, + "theoretical_loss": 3.5836252949817027, + "tokens_seen": 1214054400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192878635907723, + "loss": 2.7391, + "theoretical_loss": 3.583607539893557, + "tokens_seen": 1214119936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192778335005015, + "loss": 2.6003, + "theoretical_loss": 3.583589786032106, + "tokens_seen": 1214185472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192678034102307, + "loss": 2.8058, + "theoretical_loss": 3.583572033397197, + "tokens_seen": 1214251008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031925777331995986, + "loss": 2.8194, + "theoretical_loss": 3.5835542819886808, + "tokens_seen": 1214316544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192477432296891, + "loss": 2.5911, + "theoretical_loss": 3.5835365318064047, + "tokens_seen": 1214382080 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192377131394182, + "loss": 2.6463, + "theoretical_loss": 3.5835187828502186, + "tokens_seen": 1214447616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031922768304914746, + "loss": 2.602, + "theoretical_loss": 3.583501035119972, + "tokens_seen": 1214513152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192176529588766, + "loss": 2.4129, + "theoretical_loss": 3.583483288615514, + "tokens_seen": 1214578688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192076228686058, + "loss": 2.7214, + "theoretical_loss": 3.5834655433366933, + "tokens_seen": 1214644224 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319197592778335, + "loss": 2.5718, + "theoretical_loss": 3.583447799283359, + "tokens_seen": 1214709760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191875626880642, + "loss": 2.8878, + "theoretical_loss": 3.583430056455361, + "tokens_seen": 1214775296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031917753259779337, + "loss": 2.3432, + "theoretical_loss": 3.5834123148525485, + "tokens_seen": 1214840832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191675025075226, + "loss": 2.7502, + "theoretical_loss": 3.583394574474771, + "tokens_seen": 1214906368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031915747241725173, + "loss": 2.6352, + "theoretical_loss": 3.5833768353218773, + "tokens_seen": 1214971904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031914744232698096, + "loss": 2.4479, + "theoretical_loss": 3.583359097393717, + "tokens_seen": 1215037440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191374122367101, + "loss": 2.6745, + "theoretical_loss": 3.583341360690139, + "tokens_seen": 1215102976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191273821464393, + "loss": 2.7993, + "theoretical_loss": 3.5833236252109937, + "tokens_seen": 1215168512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191173520561685, + "loss": 2.6827, + "theoretical_loss": 3.5833058909561295, + "tokens_seen": 1215234048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191073219658977, + "loss": 2.8311, + "theoretical_loss": 3.583288157925397, + "tokens_seen": 1215299584 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031909729187562687, + "loss": 2.8424, + "theoretical_loss": 3.583270426118645, + "tokens_seen": 1215365120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031908726178535605, + "loss": 2.927, + "theoretical_loss": 3.583252695535723, + "tokens_seen": 1215430656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031907723169508523, + "loss": 2.7119, + "theoretical_loss": 3.58323496617648, + "tokens_seen": 1215496192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031906720160481447, + "loss": 2.7552, + "theoretical_loss": 3.583217238040767, + "tokens_seen": 1215561728 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031905717151454365, + "loss": 2.6251, + "theoretical_loss": 3.583199511128433, + "tokens_seen": 1215627264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 673834, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.846170425415039, + "objective/train/theoretical_loss": 3.5831817854393266, + "objective/train/tokens_used": 1236152800, + "theoretical_loss": 3.5831817854393266, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031904714142427283, + "loss": 2.8292, + "theoretical_loss": 3.5831817854393266, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031903711133400206, + "loss": 2.6614, + "theoretical_loss": 3.5831640609732993, + "tokens_seen": 1215758336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190270812437312, + "loss": 2.634, + "theoretical_loss": 3.5831463377301995, + "tokens_seen": 1215823872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190170511534604, + "loss": 2.6109, + "theoretical_loss": 3.5831286157098767, + "tokens_seen": 1215889408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031900702106318955, + "loss": 2.6929, + "theoretical_loss": 3.5831108949121817, + "tokens_seen": 1215954944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189969909729188, + "loss": 2.7014, + "theoretical_loss": 3.5830931753369635, + "tokens_seen": 1216020480 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031898696088264797, + "loss": 2.7554, + "theoretical_loss": 3.5830754569840724, + "tokens_seen": 1216086016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031897693079237715, + "loss": 2.3739, + "theoretical_loss": 3.5830577398533574, + "tokens_seen": 1216151552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031896690070210633, + "loss": 2.8485, + "theoretical_loss": 3.5830400239446694, + "tokens_seen": 1216217088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189568706118355, + "loss": 2.9264, + "theoretical_loss": 3.5830223092578573, + "tokens_seen": 1216282624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189468405215647, + "loss": 3.0185, + "theoretical_loss": 3.583004595792772, + "tokens_seen": 1216348160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031893681043129393, + "loss": 2.384, + "theoretical_loss": 3.582986883549262, + "tokens_seen": 1216413696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031892678034102306, + "loss": 2.6403, + "theoretical_loss": 3.5829691725271786, + "tokens_seen": 1216479232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189167502507523, + "loss": 2.6133, + "theoretical_loss": 3.5829514627263714, + "tokens_seen": 1216544768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189067201604814, + "loss": 2.5405, + "theoretical_loss": 3.5829337541466906, + "tokens_seen": 1216610304 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031889669007021065, + "loss": 2.6433, + "theoretical_loss": 3.5829160467879855, + "tokens_seen": 1216675840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031888665997993984, + "loss": 2.6715, + "theoretical_loss": 3.5828983406501065, + "tokens_seen": 1216741376 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318876629889669, + "loss": 2.6068, + "theoretical_loss": 3.582880635732904, + "tokens_seen": 1216806912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188665997993982, + "loss": 2.4911, + "theoretical_loss": 3.5828629320362273, + "tokens_seen": 1216872448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031885656970912743, + "loss": 2.6389, + "theoretical_loss": 3.5828452295599273, + "tokens_seen": 1216937984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031884653961885656, + "loss": 2.7881, + "theoretical_loss": 3.5828275283038544, + "tokens_seen": 1217003520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188365095285858, + "loss": 2.6524, + "theoretical_loss": 3.582809828267858, + "tokens_seen": 1217069056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188264794383149, + "loss": 2.6268, + "theoretical_loss": 3.582792129451789, + "tokens_seen": 1217134592 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031881644934804416, + "loss": 2.6762, + "theoretical_loss": 3.582774431855497, + "tokens_seen": 1217200128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031880641925777334, + "loss": 2.2686, + "theoretical_loss": 3.5827567354788323, + "tokens_seen": 1217265664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 674331, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5128471851348877, + "objective/train/theoretical_loss": 3.5827390403216457, + "objective/train/tokens_used": 1237791200, + "theoretical_loss": 3.5827390403216457, + "tokens_seen": 1217331200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187963891675025, + "loss": 2.7099, + "theoretical_loss": 3.5827390403216457, + "tokens_seen": 1217331200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187863590772317, + "loss": 2.8773, + "theoretical_loss": 3.5827213463837877, + "tokens_seen": 1217396736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187763289869609, + "loss": 2.7363, + "theoretical_loss": 3.5827036536651073, + "tokens_seen": 1217462272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031876629889669006, + "loss": 2.6986, + "theoretical_loss": 3.5826859621654568, + "tokens_seen": 1217527808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187562688064193, + "loss": 2.4538, + "theoretical_loss": 3.582668271884685, + "tokens_seen": 1217593344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187462387161484, + "loss": 2.6988, + "theoretical_loss": 3.582650582822643, + "tokens_seen": 1217658880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031873620862587766, + "loss": 2.6411, + "theoretical_loss": 3.5826328949791812, + "tokens_seen": 1217724416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187261785356068, + "loss": 2.4917, + "theoretical_loss": 3.58261520835415, + "tokens_seen": 1217789952 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318716148445336, + "loss": 2.7464, + "theoretical_loss": 3.5825975229474, + "tokens_seen": 1217855488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187061183550652, + "loss": 2.669, + "theoretical_loss": 3.582579838758782, + "tokens_seen": 1217921024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186960882647944, + "loss": 2.4795, + "theoretical_loss": 3.5825621557881457, + "tokens_seen": 1217986560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031868605817452357, + "loss": 2.6991, + "theoretical_loss": 3.5825444740353425, + "tokens_seen": 1218052096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186760280842528, + "loss": 2.6764, + "theoretical_loss": 3.5825267935002225, + "tokens_seen": 1218117632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031866599799398193, + "loss": 2.5941, + "theoretical_loss": 3.5825091141826366, + "tokens_seen": 1218183168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031865596790371116, + "loss": 2.6048, + "theoretical_loss": 3.582491436082436, + "tokens_seen": 1218248704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186459378134403, + "loss": 2.5775, + "theoretical_loss": 3.58247375919947, + "tokens_seen": 1218314240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186359077231695, + "loss": 2.7505, + "theoretical_loss": 3.582456083533591, + "tokens_seen": 1218379776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186258776328987, + "loss": 2.2671, + "theoretical_loss": 3.582438409084648, + "tokens_seen": 1218445312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186158475426279, + "loss": 2.9417, + "theoretical_loss": 3.5824207358524935, + "tokens_seen": 1218510848 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031860581745235707, + "loss": 2.6654, + "theoretical_loss": 3.582403063836977, + "tokens_seen": 1218576384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031859578736208625, + "loss": 2.6196, + "theoretical_loss": 3.5823853930379497, + "tokens_seen": 1218641920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031858575727181543, + "loss": 2.7017, + "theoretical_loss": 3.5823677234552624, + "tokens_seen": 1218707456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031857572718154467, + "loss": 2.6233, + "theoretical_loss": 3.5823500550887664, + "tokens_seen": 1218772992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185656970912738, + "loss": 2.3861, + "theoretical_loss": 3.5823323879383118, + "tokens_seen": 1218838528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031855566700100303, + "loss": 2.766, + "theoretical_loss": 3.5823147220037503, + "tokens_seen": 1218904064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 675627, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.326138973236084, + "objective/train/theoretical_loss": 3.582297057284933, + "objective/train/tokens_used": 1239429600, + "theoretical_loss": 3.582297057284933, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185456369107322, + "loss": 2.6397, + "theoretical_loss": 3.582297057284933, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185356068204614, + "loss": 2.5176, + "theoretical_loss": 3.58227939378171, + "tokens_seen": 1219035136 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031852557673019057, + "loss": 2.5311, + "theoretical_loss": 3.5822617314939325, + "tokens_seen": 1219100672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031851554663991975, + "loss": 2.6315, + "theoretical_loss": 3.582244070421452, + "tokens_seen": 1219166208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031850551654964893, + "loss": 2.5092, + "theoretical_loss": 3.5822264105641195, + "tokens_seen": 1219231744 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031849548645937817, + "loss": 2.5286, + "theoretical_loss": 3.582208751921786, + "tokens_seen": 1219297280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184854563691073, + "loss": 2.8103, + "theoretical_loss": 3.582191094494302, + "tokens_seen": 1219362816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031847542627883653, + "loss": 2.4883, + "theoretical_loss": 3.5821734382815196, + "tokens_seen": 1219428352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031846539618856566, + "loss": 2.7542, + "theoretical_loss": 3.582155783283289, + "tokens_seen": 1219493888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184553660982949, + "loss": 2.481, + "theoretical_loss": 3.5821381294994628, + "tokens_seen": 1219559424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184453360080241, + "loss": 2.7431, + "theoretical_loss": 3.582120476929891, + "tokens_seen": 1219624960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031843530591775326, + "loss": 2.4548, + "theoretical_loss": 3.5821028255744247, + "tokens_seen": 1219690496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031842527582748244, + "loss": 2.6645, + "theoretical_loss": 3.582085175432916, + "tokens_seen": 1219756032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184152457372116, + "loss": 2.363, + "theoretical_loss": 3.5820675265052158, + "tokens_seen": 1219821568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184052156469408, + "loss": 2.6981, + "theoretical_loss": 3.5820498787911754, + "tokens_seen": 1219887104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031839518555667004, + "loss": 2.688, + "theoretical_loss": 3.582032232290646, + "tokens_seen": 1219952640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031838515546639916, + "loss": 2.6851, + "theoretical_loss": 3.5820145870034796, + "tokens_seen": 1220018176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183751253761284, + "loss": 2.6927, + "theoretical_loss": 3.5819969429295266, + "tokens_seen": 1220083712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183650952858576, + "loss": 2.7519, + "theoretical_loss": 3.5819793000686397, + "tokens_seen": 1220149248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031835506519558676, + "loss": 2.5282, + "theoretical_loss": 3.581961658420669, + "tokens_seen": 1220214784 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031834503510531594, + "loss": 2.6981, + "theoretical_loss": 3.5819440179854674, + "tokens_seen": 1220280320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183350050150451, + "loss": 2.4955, + "theoretical_loss": 3.5819263787628848, + "tokens_seen": 1220345856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183249749247743, + "loss": 2.6698, + "theoretical_loss": 3.5819087407527737, + "tokens_seen": 1220411392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031831494483450354, + "loss": 2.8391, + "theoretical_loss": 3.5818911039549857, + "tokens_seen": 1220476928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183049147442327, + "loss": 2.6485, + "theoretical_loss": 3.581873468369372, + "tokens_seen": 1220542464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 676618, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6377201080322266, + "objective/train/theoretical_loss": 3.5818558339957844, + "objective/train/tokens_used": 1241068000, + "theoretical_loss": 3.5818558339957844, + "tokens_seen": 1220608000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182948846539619, + "loss": 2.5538, + "theoretical_loss": 3.5818558339957844, + "tokens_seen": 1220608000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182848545636911, + "loss": 2.7519, + "theoretical_loss": 3.581838200834075, + "tokens_seen": 1220673536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031827482447342026, + "loss": 2.6153, + "theoretical_loss": 3.581820568884094, + "tokens_seen": 1220739072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182647943831495, + "loss": 2.7711, + "theoretical_loss": 3.581802938145695, + "tokens_seen": 1220804608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182547642928786, + "loss": 2.7786, + "theoretical_loss": 3.581785308618729, + "tokens_seen": 1220870144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031824473420260786, + "loss": 2.5908, + "theoretical_loss": 3.5817676803030465, + "tokens_seen": 1220935680 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318234704112337, + "loss": 2.6733, + "theoretical_loss": 3.5817500531985007, + "tokens_seen": 1221001216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182246740220662, + "loss": 2.5456, + "theoretical_loss": 3.581732427304943, + "tokens_seen": 1221066752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182146439317954, + "loss": 2.5502, + "theoretical_loss": 3.5817148026222254, + "tokens_seen": 1221132288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182046138415246, + "loss": 2.8477, + "theoretical_loss": 3.5816971791501993, + "tokens_seen": 1221197824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031819458375125377, + "loss": 2.6389, + "theoretical_loss": 3.5816795568887168, + "tokens_seen": 1221263360 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318184553660983, + "loss": 2.6474, + "theoretical_loss": 3.58166193583763, + "tokens_seen": 1221328896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031817452357071213, + "loss": 2.6726, + "theoretical_loss": 3.5816443159967903, + "tokens_seen": 1221394432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031816449348044136, + "loss": 2.7162, + "theoretical_loss": 3.58162669736605, + "tokens_seen": 1221459968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181544633901705, + "loss": 2.629, + "theoretical_loss": 3.5816090799452613, + "tokens_seen": 1221525504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181444332998997, + "loss": 2.5372, + "theoretical_loss": 3.581591463734276, + "tokens_seen": 1221591040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181344032096289, + "loss": 2.5658, + "theoretical_loss": 3.581573848732946, + "tokens_seen": 1221656576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181243731193581, + "loss": 2.5672, + "theoretical_loss": 3.581556234941123, + "tokens_seen": 1221722112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031811434302908727, + "loss": 2.8258, + "theoretical_loss": 3.58153862235866, + "tokens_seen": 1221787648 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031810431293881645, + "loss": 2.8054, + "theoretical_loss": 3.581521010985408, + "tokens_seen": 1221853184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031809428284854563, + "loss": 2.7113, + "theoretical_loss": 3.5815034008212203, + "tokens_seen": 1221918720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031808425275827487, + "loss": 2.6263, + "theoretical_loss": 3.5814857918659477, + "tokens_seen": 1221984256 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318074222668004, + "loss": 2.9308, + "theoretical_loss": 3.5814681841194442, + "tokens_seen": 1222049792 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031806419257773323, + "loss": 2.6379, + "theoretical_loss": 3.58145057758156, + "tokens_seen": 1222115328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180541624874624, + "loss": 2.5758, + "theoretical_loss": 3.581432972252149, + "tokens_seen": 1222180864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 677215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3762056827545166, + "objective/train/theoretical_loss": 3.5814153681310623, + "objective/train/tokens_used": 1242706400, + "theoretical_loss": 3.5814153681310623, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180441323971916, + "loss": 2.7435, + "theoretical_loss": 3.5814153681310623, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031803410230692077, + "loss": 2.716, + "theoretical_loss": 3.581397765218153, + "tokens_seen": 1222311936 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031802407221664995, + "loss": 2.6065, + "theoretical_loss": 3.581380163513273, + "tokens_seen": 1222377472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031801404212637913, + "loss": 2.7569, + "theoretical_loss": 3.5813625630162744, + "tokens_seen": 1222443008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031800401203610837, + "loss": 2.7203, + "theoretical_loss": 3.58134496372701, + "tokens_seen": 1222508544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179939819458375, + "loss": 2.6444, + "theoretical_loss": 3.581327365645332, + "tokens_seen": 1222574080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031798395185556673, + "loss": 2.5914, + "theoretical_loss": 3.581309768771093, + "tokens_seen": 1222639616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031797392176529586, + "loss": 2.6799, + "theoretical_loss": 3.581292173104145, + "tokens_seen": 1222705152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179638916750251, + "loss": 2.8491, + "theoretical_loss": 3.581274578644341, + "tokens_seen": 1222770688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179538615847543, + "loss": 2.4735, + "theoretical_loss": 3.5812569853915335, + "tokens_seen": 1222836224 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031794383149448346, + "loss": 2.7763, + "theoretical_loss": 3.5812393933455744, + "tokens_seen": 1222901760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031793380140421264, + "loss": 2.55, + "theoretical_loss": 3.5812218025063167, + "tokens_seen": 1222967296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179237713139418, + "loss": 2.4893, + "theoretical_loss": 3.581204212873613, + "tokens_seen": 1223032832 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317913741223671, + "loss": 2.7899, + "theoretical_loss": 3.581186624447316, + "tokens_seen": 1223098368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031790371113340024, + "loss": 2.4937, + "theoretical_loss": 3.5811690372272773, + "tokens_seen": 1223163904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031789368104312936, + "loss": 2.8664, + "theoretical_loss": 3.581151451213351, + "tokens_seen": 1223229440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178836509528586, + "loss": 2.7589, + "theoretical_loss": 3.5811338664053887, + "tokens_seen": 1223294976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178736208625878, + "loss": 2.4983, + "theoretical_loss": 3.581116282803244, + "tokens_seen": 1223360512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031786359077231696, + "loss": 2.7072, + "theoretical_loss": 3.581098700406769, + "tokens_seen": 1223426048 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031785356068204614, + "loss": 2.6054, + "theoretical_loss": 3.581081119215816, + "tokens_seen": 1223491584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178435305917753, + "loss": 2.4573, + "theoretical_loss": 3.5810635392302395, + "tokens_seen": 1223557120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178335005015045, + "loss": 2.5697, + "theoretical_loss": 3.5810459604498908, + "tokens_seen": 1223622656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031782347041123374, + "loss": 2.4891, + "theoretical_loss": 3.5810283828746226, + "tokens_seen": 1223688192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031781344032096287, + "loss": 2.8321, + "theoretical_loss": 3.5810108065042887, + "tokens_seen": 1223753728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178034102306921, + "loss": 2.7512, + "theoretical_loss": 3.5809932313387414, + "tokens_seen": 1223819264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 678133, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7944464683532715, + "objective/train/theoretical_loss": 3.5809756573778335, + "objective/train/tokens_used": 1244344800, + "theoretical_loss": 3.5809756573778335, + "tokens_seen": 1223884800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031779338014042123, + "loss": 2.6868, + "theoretical_loss": 3.5809756573778335, + "tokens_seen": 1223884800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031778335005015046, + "loss": 2.837, + "theoretical_loss": 3.5809580846214186, + "tokens_seen": 1223950336 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031777331995987964, + "loss": 2.7034, + "theoretical_loss": 3.580940513069349, + "tokens_seen": 1224015872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177632898696088, + "loss": 2.7941, + "theoretical_loss": 3.580922942721478, + "tokens_seen": 1224081408 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317753259779338, + "loss": 2.7049, + "theoretical_loss": 3.580905373577659, + "tokens_seen": 1224146944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177432296890672, + "loss": 2.7759, + "theoretical_loss": 3.580887805637744, + "tokens_seen": 1224212480 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031773319959879637, + "loss": 2.6332, + "theoretical_loss": 3.5808702389015865, + "tokens_seen": 1224278016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177231695085256, + "loss": 2.7317, + "theoretical_loss": 3.5808526733690402, + "tokens_seen": 1224343552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031771313941825473, + "loss": 2.6885, + "theoretical_loss": 3.580835109039958, + "tokens_seen": 1224409088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031770310932798397, + "loss": 2.6962, + "theoretical_loss": 3.5808175459141918, + "tokens_seen": 1224474624 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031769307923771315, + "loss": 2.6085, + "theoretical_loss": 3.5807999839915965, + "tokens_seen": 1224540160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031768304914744233, + "loss": 2.5963, + "theoretical_loss": 3.5807824232720247, + "tokens_seen": 1224605696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176730190571715, + "loss": 2.7513, + "theoretical_loss": 3.580764863755329, + "tokens_seen": 1224671232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176629889669007, + "loss": 2.6441, + "theoretical_loss": 3.580747305441363, + "tokens_seen": 1224736768 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031765295887662987, + "loss": 2.3998, + "theoretical_loss": 3.58072974832998, + "tokens_seen": 1224802304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176429287863591, + "loss": 2.7632, + "theoretical_loss": 3.580712192421034, + "tokens_seen": 1224867840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031763289869608823, + "loss": 2.6429, + "theoretical_loss": 3.580694637714377, + "tokens_seen": 1224933376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031762286860581747, + "loss": 2.5251, + "theoretical_loss": 3.5806770842098636, + "tokens_seen": 1224998912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176128385155466, + "loss": 2.5689, + "theoretical_loss": 3.580659531907346, + "tokens_seen": 1225064448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031760280842527583, + "loss": 2.8738, + "theoretical_loss": 3.5806419808066785, + "tokens_seen": 1225129984 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317592778335005, + "loss": 2.7404, + "theoretical_loss": 3.580624430907714, + "tokens_seen": 1225195520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175827482447342, + "loss": 2.5556, + "theoretical_loss": 3.5806068822103065, + "tokens_seen": 1225261056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175727181544634, + "loss": 2.8817, + "theoretical_loss": 3.580589334714309, + "tokens_seen": 1225326592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175626880641926, + "loss": 2.719, + "theoretical_loss": 3.580571788419575, + "tokens_seen": 1225392128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175526579739218, + "loss": 2.5082, + "theoretical_loss": 3.5805542433259583, + "tokens_seen": 1225457664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 678803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4843032360076904, + "objective/train/theoretical_loss": 3.5805366994333125, + "objective/train/tokens_used": 1245983200, + "theoretical_loss": 3.5805366994333125, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031754262788365097, + "loss": 2.6552, + "theoretical_loss": 3.5805366994333125, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031753259779338015, + "loss": 2.7421, + "theoretical_loss": 3.5805191567414907, + "tokens_seen": 1225588736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031752256770310933, + "loss": 2.7794, + "theoretical_loss": 3.5805016152503466, + "tokens_seen": 1225654272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031751253761283857, + "loss": 2.7931, + "theoretical_loss": 3.580484074959734, + "tokens_seen": 1225719808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175025075225677, + "loss": 2.6884, + "theoretical_loss": 3.5804665358695074, + "tokens_seen": 1225785344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031749247743229693, + "loss": 2.5482, + "theoretical_loss": 3.5804489979795187, + "tokens_seen": 1225850880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031748244734202606, + "loss": 2.7467, + "theoretical_loss": 3.580431461289623, + "tokens_seen": 1225916416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174724172517553, + "loss": 2.7609, + "theoretical_loss": 3.5804139257996734, + "tokens_seen": 1225981952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174623871614845, + "loss": 2.5784, + "theoretical_loss": 3.580396391509524, + "tokens_seen": 1226047488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031745235707121366, + "loss": 2.6765, + "theoretical_loss": 3.580378858419028, + "tokens_seen": 1226113024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031744232698094284, + "loss": 2.7612, + "theoretical_loss": 3.58036132652804, + "tokens_seen": 1226178560 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317432296890672, + "loss": 2.5738, + "theoretical_loss": 3.5803437958364137, + "tokens_seen": 1226244096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174222668004012, + "loss": 2.6951, + "theoretical_loss": 3.5803262663440023, + "tokens_seen": 1226309632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031741223671013044, + "loss": 2.6944, + "theoretical_loss": 3.58030873805066, + "tokens_seen": 1226375168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031740220661985956, + "loss": 2.7802, + "theoretical_loss": 3.580291210956241, + "tokens_seen": 1226440704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173921765295888, + "loss": 2.5398, + "theoretical_loss": 3.5802736850605994, + "tokens_seen": 1226506240 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317382146439318, + "loss": 2.6482, + "theoretical_loss": 3.580256160363588, + "tokens_seen": 1226571776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031737211634904716, + "loss": 2.7186, + "theoretical_loss": 3.5802386368650625, + "tokens_seen": 1226637312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031736208625877634, + "loss": 2.534, + "theoretical_loss": 3.5802211145648757, + "tokens_seen": 1226702848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173520561685055, + "loss": 2.5662, + "theoretical_loss": 3.5802035934628815, + "tokens_seen": 1226768384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173420260782347, + "loss": 2.7673, + "theoretical_loss": 3.5801860735589344, + "tokens_seen": 1226833920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031733199598796394, + "loss": 2.5737, + "theoretical_loss": 3.5801685548528894, + "tokens_seen": 1226899456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031732196589769307, + "loss": 2.7677, + "theoretical_loss": 3.5801510373445993, + "tokens_seen": 1226964992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173119358074223, + "loss": 2.4772, + "theoretical_loss": 3.5801335210339182, + "tokens_seen": 1227030528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031730190571715143, + "loss": 2.6551, + "theoretical_loss": 3.5801160059207007, + "tokens_seen": 1227096064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 679787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1799423694610596, + "objective/train/theoretical_loss": 3.580098492004802, + "objective/train/tokens_used": 1247621600, + "theoretical_loss": 3.580098492004802, + "tokens_seen": 1227161600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031729187562688066, + "loss": 2.7681, + "theoretical_loss": 3.580098492004802, + "tokens_seen": 1227161600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031728184553660984, + "loss": 2.6133, + "theoretical_loss": 3.580080979286074, + "tokens_seen": 1227227136 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317271815446339, + "loss": 2.7949, + "theoretical_loss": 3.580063467764373, + "tokens_seen": 1227292672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172617853560682, + "loss": 2.4662, + "theoretical_loss": 3.5800459574395527, + "tokens_seen": 1227358208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172517552657974, + "loss": 2.7973, + "theoretical_loss": 3.5800284483114666, + "tokens_seen": 1227423744 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031724172517552657, + "loss": 2.4858, + "theoretical_loss": 3.58001094037997, + "tokens_seen": 1227489280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172316950852558, + "loss": 2.7632, + "theoretical_loss": 3.5799934336449173, + "tokens_seen": 1227554816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031722166499498493, + "loss": 2.4781, + "theoretical_loss": 3.579975928106162, + "tokens_seen": 1227620352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031721163490471417, + "loss": 2.825, + "theoretical_loss": 3.579958423763559, + "tokens_seen": 1227685888 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031720160481444335, + "loss": 2.662, + "theoretical_loss": 3.579940920616963, + "tokens_seen": 1227751424 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031719157472417253, + "loss": 2.6949, + "theoretical_loss": 3.579923418666228, + "tokens_seen": 1227816960 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171815446339017, + "loss": 2.7449, + "theoretical_loss": 3.5799059179112076, + "tokens_seen": 1227882496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171715145436309, + "loss": 2.7963, + "theoretical_loss": 3.5798884183517585, + "tokens_seen": 1227948032 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031716148445336007, + "loss": 2.7011, + "theoretical_loss": 3.5798709199877337, + "tokens_seen": 1228013568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171514543630893, + "loss": 2.8564, + "theoretical_loss": 3.5798534228189878, + "tokens_seen": 1228079104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031714142427281843, + "loss": 2.6509, + "theoretical_loss": 3.5798359268453757, + "tokens_seen": 1228144640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031713139418254767, + "loss": 2.5021, + "theoretical_loss": 3.5798184320667525, + "tokens_seen": 1228210176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171213640922768, + "loss": 2.6051, + "theoretical_loss": 3.579800938482972, + "tokens_seen": 1228275712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031711133400200603, + "loss": 2.8244, + "theoretical_loss": 3.579783446093889, + "tokens_seen": 1228341248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171013039117352, + "loss": 2.8286, + "theoretical_loss": 3.579765954899358, + "tokens_seen": 1228406784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170912738214644, + "loss": 2.8978, + "theoretical_loss": 3.579748464899234, + "tokens_seen": 1228472320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170812437311936, + "loss": 2.6003, + "theoretical_loss": 3.579730976093372, + "tokens_seen": 1228537856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170712136409228, + "loss": 2.6917, + "theoretical_loss": 3.5797134884816266, + "tokens_seen": 1228603392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031706118355065194, + "loss": 2.6362, + "theoretical_loss": 3.5796960020638524, + "tokens_seen": 1228668928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170511534603812, + "loss": 2.604, + "theoretical_loss": 3.5796785168399037, + "tokens_seen": 1228734464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 680312, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.374969482421875, + "objective/train/theoretical_loss": 3.5796610328096365, + "objective/train/tokens_used": 1249260000, + "theoretical_loss": 3.5796610328096365, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170411233701103, + "loss": 2.6887, + "theoretical_loss": 3.5796610328096365, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031703109327983954, + "loss": 2.7248, + "theoretical_loss": 3.5796435499729045, + "tokens_seen": 1228865536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170210631895687, + "loss": 2.8177, + "theoretical_loss": 3.5796260683295635, + "tokens_seen": 1228931072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170110330992979, + "loss": 2.6824, + "theoretical_loss": 3.579608587879468, + "tokens_seen": 1228996608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170010030090271, + "loss": 2.9693, + "theoretical_loss": 3.5795911086224725, + "tokens_seen": 1229062144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031699097291875626, + "loss": 2.7433, + "theoretical_loss": 3.579573630558433, + "tokens_seen": 1229127680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031698094282848544, + "loss": 2.5788, + "theoretical_loss": 3.5795561536872036, + "tokens_seen": 1229193216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169709127382147, + "loss": 2.7061, + "theoretical_loss": 3.57953867800864, + "tokens_seen": 1229258752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169608826479438, + "loss": 2.6546, + "theoretical_loss": 3.5795212035225963, + "tokens_seen": 1229324288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031695085255767304, + "loss": 2.3916, + "theoretical_loss": 3.579503730228928, + "tokens_seen": 1229389824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031694082246740217, + "loss": 2.5509, + "theoretical_loss": 3.57948625812749, + "tokens_seen": 1229455360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169307923771314, + "loss": 2.6121, + "theoretical_loss": 3.5794687872181385, + "tokens_seen": 1229520896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169207622868606, + "loss": 2.3906, + "theoretical_loss": 3.5794513175007276, + "tokens_seen": 1229586432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031691073219658976, + "loss": 2.6725, + "theoretical_loss": 3.5794338489751123, + "tokens_seen": 1229651968 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031690070210631894, + "loss": 2.7202, + "theoretical_loss": 3.579416381641148, + "tokens_seen": 1229717504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168906720160482, + "loss": 2.8365, + "theoretical_loss": 3.5793989154986905, + "tokens_seen": 1229783040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168806419257773, + "loss": 2.6397, + "theoretical_loss": 3.5793814505475945, + "tokens_seen": 1229848576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031687061183550654, + "loss": 2.7826, + "theoretical_loss": 3.5793639867877154, + "tokens_seen": 1229914112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031686058174523567, + "loss": 2.472, + "theoretical_loss": 3.5793465242189075, + "tokens_seen": 1229979648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168505516549649, + "loss": 2.6565, + "theoretical_loss": 3.579329062841028, + "tokens_seen": 1230045184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168405215646941, + "loss": 2.6165, + "theoretical_loss": 3.579311602653931, + "tokens_seen": 1230110720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031683049147442327, + "loss": 2.807, + "theoretical_loss": 3.5792941436574726, + "tokens_seen": 1230176256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031682046138415245, + "loss": 2.6677, + "theoretical_loss": 3.579276685851507, + "tokens_seen": 1230241792 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031681043129388163, + "loss": 2.5583, + "theoretical_loss": 3.5792592292358902, + "tokens_seen": 1230307328 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031680040120361086, + "loss": 2.8452, + "theoretical_loss": 3.5792417738104785, + "tokens_seen": 1230372864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 681644, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6463749408721924, + "objective/train/theoretical_loss": 3.579224319575126, + "objective/train/tokens_used": 1250898400, + "theoretical_loss": 3.579224319575126, + "tokens_seen": 1230438400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031679037111334004, + "loss": 2.5301, + "theoretical_loss": 3.579224319575126, + "tokens_seen": 1230438400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167803410230692, + "loss": 2.6175, + "theoretical_loss": 3.579206866529689, + "tokens_seen": 1230503936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167703109327984, + "loss": 2.7769, + "theoretical_loss": 3.5791894146740226, + "tokens_seen": 1230569472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167602808425276, + "loss": 2.7163, + "theoretical_loss": 3.579171964007983, + "tokens_seen": 1230635008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031675025075225677, + "loss": 2.658, + "theoretical_loss": 3.579154514531425, + "tokens_seen": 1230700544 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316740220661986, + "loss": 2.7854, + "theoretical_loss": 3.579137066244204, + "tokens_seen": 1230766080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031673019057171513, + "loss": 2.747, + "theoretical_loss": 3.5791196191461774, + "tokens_seen": 1230831616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031672016048144437, + "loss": 2.5688, + "theoretical_loss": 3.5791021732371986, + "tokens_seen": 1230897152 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031671013039117355, + "loss": 2.8399, + "theoretical_loss": 3.5790847285171243, + "tokens_seen": 1230962688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031670010030090273, + "loss": 2.5067, + "theoretical_loss": 3.5790672849858103, + "tokens_seen": 1231028224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166900702106319, + "loss": 2.7153, + "theoretical_loss": 3.579049842643112, + "tokens_seen": 1231093760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166800401203611, + "loss": 2.6044, + "theoretical_loss": 3.5790324014888855, + "tokens_seen": 1231159296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031667001003009027, + "loss": 2.719, + "theoretical_loss": 3.5790149615229856, + "tokens_seen": 1231224832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166599799398195, + "loss": 2.5149, + "theoretical_loss": 3.5789975227452695, + "tokens_seen": 1231290368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031664994984954863, + "loss": 2.7169, + "theoretical_loss": 3.5789800851555924, + "tokens_seen": 1231355904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031663991975927787, + "loss": 2.51, + "theoretical_loss": 3.5789626487538095, + "tokens_seen": 1231421440 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316629889669007, + "loss": 2.6168, + "theoretical_loss": 3.5789452135397775, + "tokens_seen": 1231486976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031661985957873623, + "loss": 2.601, + "theoretical_loss": 3.578927779513352, + "tokens_seen": 1231552512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166098294884654, + "loss": 2.7909, + "theoretical_loss": 3.578910346674389, + "tokens_seen": 1231618048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165997993981946, + "loss": 2.6013, + "theoretical_loss": 3.578892915022744, + "tokens_seen": 1231683584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165897693079238, + "loss": 2.7524, + "theoretical_loss": 3.5788754845582735, + "tokens_seen": 1231749120 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316579739217653, + "loss": 2.8369, + "theoretical_loss": 3.5788580552808336, + "tokens_seen": 1231814656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031656970912738214, + "loss": 2.6386, + "theoretical_loss": 3.5788406271902797, + "tokens_seen": 1231880192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165596790371114, + "loss": 2.6326, + "theoretical_loss": 3.578823200286468, + "tokens_seen": 1231945728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165496489468405, + "loss": 2.5907, + "theoretical_loss": 3.5788057745692554, + "tokens_seen": 1232011264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 682283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9270589351654053, + "objective/train/theoretical_loss": 3.578788350038497, + "objective/train/tokens_used": 1252536800, + "theoretical_loss": 3.578788350038497, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031653961885656974, + "loss": 2.7132, + "theoretical_loss": 3.578788350038497, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165295887662989, + "loss": 2.8002, + "theoretical_loss": 3.578770926694049, + "tokens_seen": 1232142336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165195586760281, + "loss": 2.4558, + "theoretical_loss": 3.5787535045357686, + "tokens_seen": 1232207872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165095285857573, + "loss": 2.5559, + "theoretical_loss": 3.57873608356351, + "tokens_seen": 1232273408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031649949849548646, + "loss": 2.4907, + "theoretical_loss": 3.5787186637771313, + "tokens_seen": 1232338944 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031648946840521564, + "loss": 2.6496, + "theoretical_loss": 3.5787012451764877, + "tokens_seen": 1232404480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164794383149449, + "loss": 2.7241, + "theoretical_loss": 3.578683827761436, + "tokens_seen": 1232470016 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316469408224674, + "loss": 2.7307, + "theoretical_loss": 3.578666411531832, + "tokens_seen": 1232535552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031645937813440324, + "loss": 2.5383, + "theoretical_loss": 3.578648996487532, + "tokens_seen": 1232601088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031644934804413237, + "loss": 2.729, + "theoretical_loss": 3.5786315826283923, + "tokens_seen": 1232666624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164393179538616, + "loss": 2.7778, + "theoretical_loss": 3.57861416995427, + "tokens_seen": 1232732160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164292878635908, + "loss": 2.5696, + "theoretical_loss": 3.5785967584650207, + "tokens_seen": 1232797696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031641925777331996, + "loss": 2.4963, + "theoretical_loss": 3.578579348160501, + "tokens_seen": 1232863232 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031640922768304914, + "loss": 2.4366, + "theoretical_loss": 3.5785619390405667, + "tokens_seen": 1232928768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163991975927784, + "loss": 2.8987, + "theoretical_loss": 3.5785445311050754, + "tokens_seen": 1232994304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163891675025075, + "loss": 2.3145, + "theoretical_loss": 3.578527124353883, + "tokens_seen": 1233059840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031637913741223674, + "loss": 2.6965, + "theoretical_loss": 3.5785097187868455, + "tokens_seen": 1233125376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031636910732196587, + "loss": 2.6582, + "theoretical_loss": 3.57849231440382, + "tokens_seen": 1233190912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163590772316951, + "loss": 2.5731, + "theoretical_loss": 3.5784749112046637, + "tokens_seen": 1233256448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163490471414243, + "loss": 2.5886, + "theoretical_loss": 3.578457509189232, + "tokens_seen": 1233321984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031633901705115347, + "loss": 2.6165, + "theoretical_loss": 3.578440108357382, + "tokens_seen": 1233387520 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031632898696088265, + "loss": 2.663, + "theoretical_loss": 3.57842270870897, + "tokens_seen": 1233453056 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031631895687061183, + "loss": 2.7879, + "theoretical_loss": 3.578405310243853, + "tokens_seen": 1233518592 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316308926780341, + "loss": 2.4878, + "theoretical_loss": 3.578387912961887, + "tokens_seen": 1233584128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031629889669007024, + "loss": 2.8083, + "theoretical_loss": 3.57837051686293, + "tokens_seen": 1233649664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 683327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9415130615234375, + "objective/train/theoretical_loss": 3.5783531219468374, + "objective/train/tokens_used": 1254175200, + "theoretical_loss": 3.5783531219468374, + "tokens_seen": 1233715200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031628886659979937, + "loss": 2.7898, + "theoretical_loss": 3.5783531219468374, + "tokens_seen": 1233715200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162788365095286, + "loss": 2.555, + "theoretical_loss": 3.5783357282134665, + "tokens_seen": 1233780736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031626880641925773, + "loss": 2.7074, + "theoretical_loss": 3.578318335662675, + "tokens_seen": 1233846272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031625877632898697, + "loss": 2.7915, + "theoretical_loss": 3.5783009442943174, + "tokens_seen": 1233911808 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031624874623871615, + "loss": 2.6769, + "theoretical_loss": 3.5782835541082525, + "tokens_seen": 1233977344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031623871614844533, + "loss": 2.595, + "theoretical_loss": 3.5782661651043366, + "tokens_seen": 1234042880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162286860581745, + "loss": 2.6416, + "theoretical_loss": 3.5782487772824263, + "tokens_seen": 1234108416 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031621865596790375, + "loss": 2.8331, + "theoretical_loss": 3.5782313906423786, + "tokens_seen": 1234173952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162086258776329, + "loss": 2.4344, + "theoretical_loss": 3.5782140051840505, + "tokens_seen": 1234239488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161985957873621, + "loss": 2.4601, + "theoretical_loss": 3.578196620907299, + "tokens_seen": 1234305024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031618856569709124, + "loss": 2.7753, + "theoretical_loss": 3.578179237811981, + "tokens_seen": 1234370560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031617853560682047, + "loss": 2.722, + "theoretical_loss": 3.578161855897953, + "tokens_seen": 1234436096 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031616850551654965, + "loss": 2.5857, + "theoretical_loss": 3.5781444751650735, + "tokens_seen": 1234501632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031615847542627883, + "loss": 2.5077, + "theoretical_loss": 3.5781270956131976, + "tokens_seen": 1234567168 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316148445336008, + "loss": 2.5656, + "theoretical_loss": 3.5781097172421834, + "tokens_seen": 1234632704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161384152457372, + "loss": 2.5073, + "theoretical_loss": 3.5780923400518883, + "tokens_seen": 1234698240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161283851554664, + "loss": 2.6107, + "theoretical_loss": 3.5780749640421687, + "tokens_seen": 1234763776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161183550651956, + "loss": 2.5824, + "theoretical_loss": 3.5780575892128823, + "tokens_seen": 1234829312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031610832497492474, + "loss": 2.631, + "theoretical_loss": 3.5780402155638855, + "tokens_seen": 1234894848 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316098294884654, + "loss": 2.4193, + "theoretical_loss": 3.5780228430950363, + "tokens_seen": 1234960384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160882647943831, + "loss": 2.7855, + "theoretical_loss": 3.5780054718061916, + "tokens_seen": 1235025920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031607823470411234, + "loss": 2.6025, + "theoretical_loss": 3.5779881016972093, + "tokens_seen": 1235091456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160682046138415, + "loss": 2.8302, + "theoretical_loss": 3.577970732767945, + "tokens_seen": 1235156992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160581745235707, + "loss": 2.5224, + "theoretical_loss": 3.5779533650182573, + "tokens_seen": 1235222528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031604814443329994, + "loss": 2.7319, + "theoretical_loss": 3.5779359984480035, + "tokens_seen": 1235288064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 684175, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.004976272583008, + "objective/train/theoretical_loss": 3.5779186330570405, + "objective/train/tokens_used": 1255813600, + "theoretical_loss": 3.5779186330570405, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160381143430291, + "loss": 2.6376, + "theoretical_loss": 3.5779186330570405, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160280842527583, + "loss": 2.6485, + "theoretical_loss": 3.577901268845226, + "tokens_seen": 1235419136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160180541624875, + "loss": 2.6592, + "theoretical_loss": 3.5778839058124166, + "tokens_seen": 1235484672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031600802407221666, + "loss": 2.5898, + "theoretical_loss": 3.577866543958471, + "tokens_seen": 1235550208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031599799398194584, + "loss": 2.7968, + "theoretical_loss": 3.577849183283245, + "tokens_seen": 1235615744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159879638916751, + "loss": 2.7619, + "theoretical_loss": 3.577831823786598, + "tokens_seen": 1235681280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159779338014042, + "loss": 2.6657, + "theoretical_loss": 3.577814465468386, + "tokens_seen": 1235746816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031596790371113344, + "loss": 2.5929, + "theoretical_loss": 3.5777971083284674, + "tokens_seen": 1235812352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031595787362086257, + "loss": 2.6074, + "theoretical_loss": 3.577779752366699, + "tokens_seen": 1235877888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159478435305918, + "loss": 2.5263, + "theoretical_loss": 3.577762397582939, + "tokens_seen": 1235943424 + }, + { + "epoch": 0.37, + "learning_rate": 0.000315937813440321, + "loss": 2.5976, + "theoretical_loss": 3.5777450439770444, + "tokens_seen": 1236008960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031592778335005016, + "loss": 2.7342, + "theoretical_loss": 3.5777276915488736, + "tokens_seen": 1236074496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031591775325977934, + "loss": 2.6336, + "theoretical_loss": 3.5777103402982835, + "tokens_seen": 1236140032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159077231695086, + "loss": 2.8675, + "theoretical_loss": 3.577692990225132, + "tokens_seen": 1236205568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158976930792377, + "loss": 2.791, + "theoretical_loss": 3.577675641329277, + "tokens_seen": 1236271104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031588766298896694, + "loss": 2.7231, + "theoretical_loss": 3.5776582936105754, + "tokens_seen": 1236336640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031587763289869607, + "loss": 2.575, + "theoretical_loss": 3.577640947068886, + "tokens_seen": 1236402176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158676028084253, + "loss": 2.8744, + "theoretical_loss": 3.577623601704066, + "tokens_seen": 1236467712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158575727181545, + "loss": 2.7638, + "theoretical_loss": 3.5776062575159733, + "tokens_seen": 1236533248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031584754262788367, + "loss": 2.7146, + "theoretical_loss": 3.577588914504466, + "tokens_seen": 1236598784 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031583751253761285, + "loss": 2.8056, + "theoretical_loss": 3.5775715726694015, + "tokens_seen": 1236664320 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031582748244734203, + "loss": 2.5687, + "theoretical_loss": 3.5775542320106375, + "tokens_seen": 1236729856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158174523570712, + "loss": 2.8633, + "theoretical_loss": 3.577536892528032, + "tokens_seen": 1236795392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031580742226680044, + "loss": 2.4725, + "theoretical_loss": 3.577519554221444, + "tokens_seen": 1236860928 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031579739217652957, + "loss": 2.7498, + "theoretical_loss": 3.5775022170907302, + "tokens_seen": 1236926464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 684175, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.575035572052002, + "objective/train/theoretical_loss": 3.5774848811357485, + "objective/train/tokens_used": 1257452000, + "theoretical_loss": 3.5774848811357485, + "tokens_seen": 1236992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157873620862588, + "loss": 2.7459, + "theoretical_loss": 3.5774848811357485, + "tokens_seen": 1236992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031577733199598793, + "loss": 2.5454, + "theoretical_loss": 3.5774675463563574, + "tokens_seen": 1237057536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031576730190571717, + "loss": 2.6331, + "theoretical_loss": 3.5774502127524146, + "tokens_seen": 1237123072 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031575727181544635, + "loss": 2.7631, + "theoretical_loss": 3.5774328803237787, + "tokens_seen": 1237188608 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031574724172517553, + "loss": 2.841, + "theoretical_loss": 3.577415549070307, + "tokens_seen": 1237254144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157372116349047, + "loss": 2.6371, + "theoretical_loss": 3.577398218991858, + "tokens_seen": 1237319680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031572718154463395, + "loss": 2.8772, + "theoretical_loss": 3.57738089008829, + "tokens_seen": 1237385216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157171514543631, + "loss": 2.8408, + "theoretical_loss": 3.577363562359461, + "tokens_seen": 1237450752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003157071213640923, + "loss": 2.6924, + "theoretical_loss": 3.5773462358052286, + "tokens_seen": 1237516288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031569709127382144, + "loss": 2.6864, + "theoretical_loss": 3.5773289104254515, + "tokens_seen": 1237581824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031568706118355067, + "loss": 2.7785, + "theoretical_loss": 3.577311586219988, + "tokens_seen": 1237647360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031567703109327985, + "loss": 2.5432, + "theoretical_loss": 3.577294263188696, + "tokens_seen": 1237712896 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031566700100300903, + "loss": 2.5293, + "theoretical_loss": 3.577276941331434, + "tokens_seen": 1237778432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156569709127382, + "loss": 2.6125, + "theoretical_loss": 3.57725962064806, + "tokens_seen": 1237843968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156469408224674, + "loss": 2.5755, + "theoretical_loss": 3.5772423011384324, + "tokens_seen": 1237909504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156369107321966, + "loss": 2.6375, + "theoretical_loss": 3.5772249828024103, + "tokens_seen": 1237975040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156268806419258, + "loss": 2.4348, + "theoretical_loss": 3.577207665639851, + "tokens_seen": 1238040576 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031561685055165494, + "loss": 2.7653, + "theoretical_loss": 3.5771903496506128, + "tokens_seen": 1238106112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156068204613842, + "loss": 2.8229, + "theoretical_loss": 3.577173034834555, + "tokens_seen": 1238171648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155967903711133, + "loss": 2.6971, + "theoretical_loss": 3.5771557211915352, + "tokens_seen": 1238237184 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031558676028084254, + "loss": 2.7564, + "theoretical_loss": 3.5771384087214124, + "tokens_seen": 1238302720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155767301905717, + "loss": 2.4933, + "theoretical_loss": 3.577121097424045, + "tokens_seen": 1238368256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155667001003009, + "loss": 2.5807, + "theoretical_loss": 3.5771037872992917, + "tokens_seen": 1238433792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155566700100301, + "loss": 2.5816, + "theoretical_loss": 3.57708647834701, + "tokens_seen": 1238499328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155466399197593, + "loss": 2.9417, + "theoretical_loss": 3.5770691705670594, + "tokens_seen": 1238564864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 685182, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7375147342681885, + "objective/train/theoretical_loss": 3.5770518639592983, + "objective/train/tokens_used": 1259090400, + "theoretical_loss": 3.5770518639592983, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031553660982948844, + "loss": 2.6052, + "theoretical_loss": 3.5770518639592983, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155265797392177, + "loss": 2.7781, + "theoretical_loss": 3.577034558523585, + "tokens_seen": 1238695936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155165496489468, + "loss": 2.8369, + "theoretical_loss": 3.577017254259779, + "tokens_seen": 1238761472 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031550651955867604, + "loss": 2.7518, + "theoretical_loss": 3.5769999511677377, + "tokens_seen": 1238827008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154964894684052, + "loss": 2.5688, + "theoretical_loss": 3.576982649247321, + "tokens_seen": 1238892544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154864593781344, + "loss": 2.7044, + "theoretical_loss": 3.576965348498386, + "tokens_seen": 1238958080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154764292878636, + "loss": 2.9206, + "theoretical_loss": 3.5769480489207934, + "tokens_seen": 1239023616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031546639919759277, + "loss": 2.8187, + "theoretical_loss": 3.5769307505144003, + "tokens_seen": 1239089152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031545636910732195, + "loss": 2.5674, + "theoretical_loss": 3.576913453279066, + "tokens_seen": 1239154688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154463390170512, + "loss": 2.7482, + "theoretical_loss": 3.57689615721465, + "tokens_seen": 1239220224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154363089267803, + "loss": 2.8608, + "theoretical_loss": 3.57687886232101, + "tokens_seen": 1239285760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031542627883650954, + "loss": 2.6337, + "theoretical_loss": 3.5768615685980056, + "tokens_seen": 1239351296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154162487462387, + "loss": 2.7525, + "theoretical_loss": 3.5768442760454953, + "tokens_seen": 1239416832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154062186559679, + "loss": 2.6064, + "theoretical_loss": 3.5768269846633385, + "tokens_seen": 1239482368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153961885656971, + "loss": 2.6995, + "theoretical_loss": 3.576809694451393, + "tokens_seen": 1239547904 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031538615847542627, + "loss": 2.5633, + "theoretical_loss": 3.5767924054095195, + "tokens_seen": 1239613440 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031537612838515545, + "loss": 2.7152, + "theoretical_loss": 3.5767751175375753, + "tokens_seen": 1239678976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153660982948847, + "loss": 2.8287, + "theoretical_loss": 3.5767578308354198, + "tokens_seen": 1239744512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153560682046138, + "loss": 2.8507, + "theoretical_loss": 3.5767405453029126, + "tokens_seen": 1239810048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031534603811434305, + "loss": 2.799, + "theoretical_loss": 3.5767232609399127, + "tokens_seen": 1239875584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153360080240722, + "loss": 2.725, + "theoretical_loss": 3.576705977746278, + "tokens_seen": 1239941120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153259779338014, + "loss": 2.6619, + "theoretical_loss": 3.5766886957218693, + "tokens_seen": 1240006656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153159478435306, + "loss": 2.8006, + "theoretical_loss": 3.576671414866544, + "tokens_seen": 1240072192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031530591775325977, + "loss": 2.6383, + "theoretical_loss": 3.5766541351801626, + "tokens_seen": 1240137728 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315295887662989, + "loss": 2.5801, + "theoretical_loss": 3.576636856662584, + "tokens_seen": 1240203264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 685844, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0006914138793945, + "objective/train/theoretical_loss": 3.5766195793136664, + "objective/train/tokens_used": 1260728800, + "theoretical_loss": 3.5766195793136664, + "tokens_seen": 1240268800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031528585757271813, + "loss": 2.4128, + "theoretical_loss": 3.5766195793136664, + "tokens_seen": 1240268800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031527582748244737, + "loss": 2.6779, + "theoretical_loss": 3.5766023031332703, + "tokens_seen": 1240334336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031526579739217655, + "loss": 2.8779, + "theoretical_loss": 3.5765850281212543, + "tokens_seen": 1240399872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031525576730190573, + "loss": 2.5602, + "theoretical_loss": 3.5765677542774776, + "tokens_seen": 1240465408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152457372116349, + "loss": 2.7543, + "theoretical_loss": 3.5765504816018, + "tokens_seen": 1240530944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031523570712136415, + "loss": 2.7578, + "theoretical_loss": 3.57653321009408, + "tokens_seen": 1240596480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152256770310933, + "loss": 2.6434, + "theoretical_loss": 3.5765159397541773, + "tokens_seen": 1240662016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152156469408225, + "loss": 2.6226, + "theoretical_loss": 3.5764986705819517, + "tokens_seen": 1240727552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031520561685055164, + "loss": 2.9745, + "theoretical_loss": 3.5764814025772615, + "tokens_seen": 1240793088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031519558676028087, + "loss": 2.8188, + "theoretical_loss": 3.5764641357399674, + "tokens_seen": 1240858624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031518555667001005, + "loss": 2.7181, + "theoretical_loss": 3.576446870069928, + "tokens_seen": 1240924160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031517552657973924, + "loss": 2.8341, + "theoretical_loss": 3.576429605567003, + "tokens_seen": 1240989696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151654964894684, + "loss": 2.7724, + "theoretical_loss": 3.576412342231052, + "tokens_seen": 1241055232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151554663991976, + "loss": 2.6043, + "theoretical_loss": 3.576395080061934, + "tokens_seen": 1241120768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151454363089268, + "loss": 2.6358, + "theoretical_loss": 3.576377819059509, + "tokens_seen": 1241186304 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315135406218656, + "loss": 2.8804, + "theoretical_loss": 3.5763605592236365, + "tokens_seen": 1241251840 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031512537612838514, + "loss": 2.7021, + "theoretical_loss": 3.5763433005541767, + "tokens_seen": 1241317376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151153460381144, + "loss": 2.3519, + "theoretical_loss": 3.5763260430509876, + "tokens_seen": 1241382912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151053159478435, + "loss": 2.8287, + "theoretical_loss": 3.57630878671393, + "tokens_seen": 1241448448 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031509528585757274, + "loss": 2.3984, + "theoretical_loss": 3.5762915315428634, + "tokens_seen": 1241513984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150852557673019, + "loss": 2.6735, + "theoretical_loss": 3.576274277537647, + "tokens_seen": 1241579520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150752256770311, + "loss": 2.7345, + "theoretical_loss": 3.5762570246981413, + "tokens_seen": 1241645056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150651955867603, + "loss": 2.6689, + "theoretical_loss": 3.5762397730242053, + "tokens_seen": 1241710592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150551654964895, + "loss": 2.8214, + "theoretical_loss": 3.5762225225156987, + "tokens_seen": 1241776128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031504513540621864, + "loss": 2.8196, + "theoretical_loss": 3.576205273172482, + "tokens_seen": 1241841664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 687001, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8483855724334717, + "objective/train/theoretical_loss": 3.5761880249944147, + "objective/train/tokens_used": 1262367200, + "theoretical_loss": 3.5761880249944147, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150351053159479, + "loss": 2.6797, + "theoretical_loss": 3.5761880249944147, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315025075225677, + "loss": 2.8243, + "theoretical_loss": 3.576170777981356, + "tokens_seen": 1241972736 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031501504513540624, + "loss": 2.812, + "theoretical_loss": 3.5761535321331666, + "tokens_seen": 1242038272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150050150451354, + "loss": 2.6161, + "theoretical_loss": 3.5761362874497062, + "tokens_seen": 1242103808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149949849548646, + "loss": 2.5499, + "theoretical_loss": 3.5761190439308344, + "tokens_seen": 1242169344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149849548645938, + "loss": 2.7484, + "theoretical_loss": 3.576101801576411, + "tokens_seen": 1242234880 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031497492477432297, + "loss": 2.7865, + "theoretical_loss": 3.5760845603862963, + "tokens_seen": 1242300416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031496489468405215, + "loss": 2.7135, + "theoretical_loss": 3.57606732036035, + "tokens_seen": 1242365952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149548645937814, + "loss": 2.8205, + "theoretical_loss": 3.576050081498432, + "tokens_seen": 1242431488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149448345035105, + "loss": 2.6539, + "theoretical_loss": 3.5760328438004025, + "tokens_seen": 1242497024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031493480441323974, + "loss": 2.5831, + "theoretical_loss": 3.5760156072661218, + "tokens_seen": 1242562560 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149247743229689, + "loss": 2.7412, + "theoretical_loss": 3.57599837189545, + "tokens_seen": 1242628096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149147442326981, + "loss": 2.7729, + "theoretical_loss": 3.575981137688246, + "tokens_seen": 1242693632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149047141424273, + "loss": 2.5824, + "theoretical_loss": 3.5759639046443716, + "tokens_seen": 1242759168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031489468405215647, + "loss": 2.6411, + "theoretical_loss": 3.575946672763686, + "tokens_seen": 1242824704 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031488465396188565, + "loss": 2.7375, + "theoretical_loss": 3.575929442046049, + "tokens_seen": 1242890240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148746238716149, + "loss": 2.7509, + "theoretical_loss": 3.575912212491322, + "tokens_seen": 1242955776 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314864593781344, + "loss": 2.7655, + "theoretical_loss": 3.575894984099364, + "tokens_seen": 1243021312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031485456369107325, + "loss": 2.8217, + "theoretical_loss": 3.575877756870036, + "tokens_seen": 1243086848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148445336008024, + "loss": 2.6609, + "theoretical_loss": 3.575860530803198, + "tokens_seen": 1243152384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148345035105316, + "loss": 2.8145, + "theoretical_loss": 3.57584330589871, + "tokens_seen": 1243217920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148244734202608, + "loss": 2.6594, + "theoretical_loss": 3.575826082156432, + "tokens_seen": 1243283456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031481444332998997, + "loss": 2.5629, + "theoretical_loss": 3.5758088595762256, + "tokens_seen": 1243348992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031480441323971915, + "loss": 2.5686, + "theoretical_loss": 3.57579163815795, + "tokens_seen": 1243414528 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031479438314944833, + "loss": 2.7092, + "theoretical_loss": 3.5757744179014668, + "tokens_seen": 1243480064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 688085, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8447015285491943, + "objective/train/theoretical_loss": 3.575757198806635, + "objective/train/tokens_used": 1264005600, + "theoretical_loss": 3.575757198806635, + "tokens_seen": 1243545600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147843530591775, + "loss": 2.9024, + "theoretical_loss": 3.575757198806635, + "tokens_seen": 1243545600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031477432296890675, + "loss": 2.7652, + "theoretical_loss": 3.5757399808733155, + "tokens_seen": 1243611136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147642928786359, + "loss": 2.8517, + "theoretical_loss": 3.575722764101369, + "tokens_seen": 1243676672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147542627883651, + "loss": 2.7982, + "theoretical_loss": 3.575705548490656, + "tokens_seen": 1243742208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147442326980943, + "loss": 2.4399, + "theoretical_loss": 3.5756883340410366, + "tokens_seen": 1243807744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147342026078235, + "loss": 2.4896, + "theoretical_loss": 3.5756711207523715, + "tokens_seen": 1243873280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031472417251755266, + "loss": 2.6552, + "theoretical_loss": 3.5756539086245214, + "tokens_seen": 1243938816 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031471414242728184, + "loss": 2.649, + "theoretical_loss": 3.575636697657347, + "tokens_seen": 1244004352 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314704112337011, + "loss": 2.7111, + "theoretical_loss": 3.5756194878507084, + "tokens_seen": 1244069888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031469408224674025, + "loss": 2.6986, + "theoretical_loss": 3.5756022792044666, + "tokens_seen": 1244135424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146840521564694, + "loss": 2.7551, + "theoretical_loss": 3.5755850717184816, + "tokens_seen": 1244200960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146740220661986, + "loss": 2.7036, + "theoretical_loss": 3.575567865392615, + "tokens_seen": 1244266496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031466399197592774, + "loss": 2.7044, + "theoretical_loss": 3.5755506602267273, + "tokens_seen": 1244332032 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314653961885657, + "loss": 2.715, + "theoretical_loss": 3.575533456220678, + "tokens_seen": 1244397568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031464393179538616, + "loss": 2.6174, + "theoretical_loss": 3.57551625337433, + "tokens_seen": 1244463104 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031463390170511534, + "loss": 2.6853, + "theoretical_loss": 3.575499051687542, + "tokens_seen": 1244528640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146238716148445, + "loss": 2.8587, + "theoretical_loss": 3.575481851160176, + "tokens_seen": 1244594176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146138415245737, + "loss": 2.8025, + "theoretical_loss": 3.5754646517920925, + "tokens_seen": 1244659712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146038114343029, + "loss": 2.8209, + "theoretical_loss": 3.575447453583152, + "tokens_seen": 1244725248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145937813440321, + "loss": 2.5335, + "theoretical_loss": 3.575430256533216, + "tokens_seen": 1244790784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031458375125376125, + "loss": 2.5279, + "theoretical_loss": 3.575413060642145, + "tokens_seen": 1244856320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145737211634905, + "loss": 2.6258, + "theoretical_loss": 3.5753958659097993, + "tokens_seen": 1244921856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145636910732197, + "loss": 2.6978, + "theoretical_loss": 3.5753786723360412, + "tokens_seen": 1244987392 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031455366098294884, + "loss": 2.6572, + "theoretical_loss": 3.5753614799207307, + "tokens_seen": 1245052928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145436308926781, + "loss": 2.9123, + "theoretical_loss": 3.5753442886637288, + "tokens_seen": 1245118464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 688769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5056591033935547, + "objective/train/theoretical_loss": 3.5753270985648973, + "objective/train/tokens_used": 1265644000, + "theoretical_loss": 3.5753270985648973, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145336008024072, + "loss": 2.555, + "theoretical_loss": 3.5753270985648973, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031452357071213644, + "loss": 2.5788, + "theoretical_loss": 3.575309909624096, + "tokens_seen": 1245249536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145135406218656, + "loss": 2.7804, + "theoretical_loss": 3.5752927218411865, + "tokens_seen": 1245315072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145035105315948, + "loss": 2.6175, + "theoretical_loss": 3.57527553521603, + "tokens_seen": 1245380608 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314493480441324, + "loss": 2.5547, + "theoretical_loss": 3.5752583497484878, + "tokens_seen": 1245446144 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031448345035105317, + "loss": 2.6196, + "theoretical_loss": 3.5752411654384204, + "tokens_seen": 1245511680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031447342026078235, + "loss": 2.6699, + "theoretical_loss": 3.5752239822856895, + "tokens_seen": 1245577216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144633901705116, + "loss": 2.7614, + "theoretical_loss": 3.5752068002901565, + "tokens_seen": 1245642752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144533600802407, + "loss": 2.7317, + "theoretical_loss": 3.5751896194516815, + "tokens_seen": 1245708288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031444332998996994, + "loss": 2.5388, + "theoretical_loss": 3.5751724397701263, + "tokens_seen": 1245773824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144332998996991, + "loss": 2.6477, + "theoretical_loss": 3.575155261245353, + "tokens_seen": 1245839360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144232698094283, + "loss": 2.7932, + "theoretical_loss": 3.5751380838772215, + "tokens_seen": 1245904896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144132397191575, + "loss": 2.5666, + "theoretical_loss": 3.5751209076655934, + "tokens_seen": 1245970432 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031440320962888667, + "loss": 2.5904, + "theoretical_loss": 3.575103732610331, + "tokens_seen": 1246035968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031439317953861585, + "loss": 2.7356, + "theoretical_loss": 3.575086558711294, + "tokens_seen": 1246101504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143831494483451, + "loss": 2.6404, + "theoretical_loss": 3.5750693859683453, + "tokens_seen": 1246167040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143731193580742, + "loss": 2.5866, + "theoretical_loss": 3.5750522143813455, + "tokens_seen": 1246232576 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031436308926780345, + "loss": 2.6652, + "theoretical_loss": 3.575035043950156, + "tokens_seen": 1246298112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143530591775326, + "loss": 2.7431, + "theoretical_loss": 3.5750178746746384, + "tokens_seen": 1246363648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143430290872618, + "loss": 2.7738, + "theoretical_loss": 3.5750007065546545, + "tokens_seen": 1246429184 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314332998996991, + "loss": 2.8112, + "theoretical_loss": 3.574983539590065, + "tokens_seen": 1246494720 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031432296890672017, + "loss": 2.6557, + "theoretical_loss": 3.5749663737807325, + "tokens_seen": 1246560256 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031431293881644935, + "loss": 2.7727, + "theoretical_loss": 3.574949209126517, + "tokens_seen": 1246625792 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031430290872617853, + "loss": 2.7943, + "theoretical_loss": 3.574932045627281, + "tokens_seen": 1246691328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142928786359077, + "loss": 2.6418, + "theoretical_loss": 3.5749148832828865, + "tokens_seen": 1246756864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 689881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9862053394317627, + "objective/train/theoretical_loss": 3.5748977220931946, + "objective/train/tokens_used": 1267282400, + "theoretical_loss": 3.5748977220931946, + "tokens_seen": 1246822400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031428284854563695, + "loss": 2.6589, + "theoretical_loss": 3.5748977220931946, + "tokens_seen": 1246822400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142728184553661, + "loss": 2.9614, + "theoretical_loss": 3.5748805620580666, + "tokens_seen": 1246887936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142627883650953, + "loss": 2.4722, + "theoretical_loss": 3.5748634031773645, + "tokens_seen": 1246953472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142527582748245, + "loss": 2.8278, + "theoretical_loss": 3.5748462454509498, + "tokens_seen": 1247019008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142427281845537, + "loss": 2.5662, + "theoretical_loss": 3.5748290888786842, + "tokens_seen": 1247084544 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031423269809428286, + "loss": 2.7926, + "theoretical_loss": 3.5748119334604302, + "tokens_seen": 1247150080 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031422266800401204, + "loss": 2.7912, + "theoretical_loss": 3.574794779196048, + "tokens_seen": 1247215616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142126379137412, + "loss": 2.6479, + "theoretical_loss": 3.574777626085401, + "tokens_seen": 1247281152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031420260782347045, + "loss": 2.837, + "theoretical_loss": 3.57476047412835, + "tokens_seen": 1247346688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141925777331996, + "loss": 2.6822, + "theoretical_loss": 3.5747433233247565, + "tokens_seen": 1247412224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141825476429288, + "loss": 2.5263, + "theoretical_loss": 3.5747261736744838, + "tokens_seen": 1247477760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031417251755265794, + "loss": 2.7709, + "theoretical_loss": 3.574709025177392, + "tokens_seen": 1247543296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141624874623872, + "loss": 2.5264, + "theoretical_loss": 3.5746918778333443, + "tokens_seen": 1247608832 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031415245737211636, + "loss": 2.6754, + "theoretical_loss": 3.574674731642202, + "tokens_seen": 1247674368 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031414242728184554, + "loss": 2.8169, + "theoretical_loss": 3.574657586603827, + "tokens_seen": 1247739904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141323971915747, + "loss": 2.8516, + "theoretical_loss": 3.5746404427180813, + "tokens_seen": 1247805440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141223671013039, + "loss": 2.9212, + "theoretical_loss": 3.574623299984827, + "tokens_seen": 1247870976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141123370110331, + "loss": 2.7596, + "theoretical_loss": 3.5746061584039266, + "tokens_seen": 1247936512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141023069207623, + "loss": 2.8479, + "theoretical_loss": 3.5745890179752413, + "tokens_seen": 1248002048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031409227683049145, + "loss": 2.6303, + "theoretical_loss": 3.5745718786986336, + "tokens_seen": 1248067584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140822467402207, + "loss": 2.9746, + "theoretical_loss": 3.5745547405739657, + "tokens_seen": 1248133120 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031407221664994986, + "loss": 2.7913, + "theoretical_loss": 3.5745376036010987, + "tokens_seen": 1248198656 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031406218655967904, + "loss": 2.8253, + "theoretical_loss": 3.5745204677798963, + "tokens_seen": 1248264192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140521564694082, + "loss": 2.7664, + "theoretical_loss": 3.5745033331102194, + "tokens_seen": 1248329728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140421263791374, + "loss": 2.5198, + "theoretical_loss": 3.5744861995919304, + "tokens_seen": 1248395264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 690418, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.728623151779175, + "objective/train/theoretical_loss": 3.574469067224892, + "objective/train/tokens_used": 1268920800, + "theoretical_loss": 3.574469067224892, + "tokens_seen": 1248460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140320962888666, + "loss": 2.7498, + "theoretical_loss": 3.574469067224892, + "tokens_seen": 1248460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140220661985958, + "loss": 2.722, + "theoretical_loss": 3.574451936008966, + "tokens_seen": 1248526336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031401203610832495, + "loss": 2.7592, + "theoretical_loss": 3.5744348059440147, + "tokens_seen": 1248591872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140020060180542, + "loss": 2.8308, + "theoretical_loss": 3.5744176770299, + "tokens_seen": 1248657408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139919759277833, + "loss": 2.6995, + "theoretical_loss": 3.5744005492664854, + "tokens_seen": 1248722944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031398194583751255, + "loss": 2.8554, + "theoretical_loss": 3.574383422653632, + "tokens_seen": 1248788480 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031397191574724173, + "loss": 2.8966, + "theoretical_loss": 3.5743662971912027, + "tokens_seen": 1248854016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139618856569709, + "loss": 2.6512, + "theoretical_loss": 3.5743491728790593, + "tokens_seen": 1248919552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139518555667001, + "loss": 2.6799, + "theoretical_loss": 3.574332049717065, + "tokens_seen": 1248985088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139418254764293, + "loss": 2.5296, + "theoretical_loss": 3.574314927705082, + "tokens_seen": 1249050624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031393179538615845, + "loss": 2.6058, + "theoretical_loss": 3.5742978068429716, + "tokens_seen": 1249116160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139217652958877, + "loss": 2.6187, + "theoretical_loss": 3.574280687130598, + "tokens_seen": 1249181696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139117352056168, + "loss": 2.5322, + "theoretical_loss": 3.5742635685678223, + "tokens_seen": 1249247232 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031390170511534605, + "loss": 2.7232, + "theoretical_loss": 3.574246451154508, + "tokens_seen": 1249312768 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031389167502507523, + "loss": 2.7474, + "theoretical_loss": 3.574229334890517, + "tokens_seen": 1249378304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138816449348044, + "loss": 2.6216, + "theoretical_loss": 3.5742122197757125, + "tokens_seen": 1249443840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138716148445336, + "loss": 2.5586, + "theoretical_loss": 3.574195105809956, + "tokens_seen": 1249509376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138615847542628, + "loss": 2.4887, + "theoretical_loss": 3.5741779929931106, + "tokens_seen": 1249574912 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031385155466399196, + "loss": 2.6734, + "theoretical_loss": 3.5741608813250396, + "tokens_seen": 1249640448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138415245737212, + "loss": 2.8049, + "theoretical_loss": 3.5741437708056045, + "tokens_seen": 1249705984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138314944834503, + "loss": 2.5464, + "theoretical_loss": 3.5741266614346685, + "tokens_seen": 1249771520 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031382146439317955, + "loss": 2.6368, + "theoretical_loss": 3.574109553212095, + "tokens_seen": 1249837056 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031381143430290873, + "loss": 2.5456, + "theoretical_loss": 3.5740924461377457, + "tokens_seen": 1249902592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138014042126379, + "loss": 2.5049, + "theoretical_loss": 3.5740753402114835, + "tokens_seen": 1249968128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031379137412236715, + "loss": 2.5967, + "theoretical_loss": 3.5740582354331716, + "tokens_seen": 1250033664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 691448, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7786600589752197, + "objective/train/theoretical_loss": 3.574041131802672, + "objective/train/tokens_used": 1270559200, + "theoretical_loss": 3.574041131802672, + "tokens_seen": 1250099200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137813440320963, + "loss": 2.703, + "theoretical_loss": 3.574041131802672, + "tokens_seen": 1250099200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137713139418255, + "loss": 2.5466, + "theoretical_loss": 3.5740240293198484, + "tokens_seen": 1250164736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137612838515547, + "loss": 2.8031, + "theoretical_loss": 3.574006927984563, + "tokens_seen": 1250230272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137512537612839, + "loss": 2.6104, + "theoretical_loss": 3.5739898277966793, + "tokens_seen": 1250295808 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031374122367101306, + "loss": 2.7823, + "theoretical_loss": 3.5739727287560594, + "tokens_seen": 1250361344 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031373119358074224, + "loss": 2.7496, + "theoretical_loss": 3.5739556308625673, + "tokens_seen": 1250426880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137211634904714, + "loss": 2.7246, + "theoretical_loss": 3.5739385341160643, + "tokens_seen": 1250492416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031371113340020065, + "loss": 2.6489, + "theoretical_loss": 3.5739214385164146, + "tokens_seen": 1250557952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137011033099298, + "loss": 2.5355, + "theoretical_loss": 3.573904344063481, + "tokens_seen": 1250623488 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313691073219659, + "loss": 2.7299, + "theoretical_loss": 3.573887250757126, + "tokens_seen": 1250689024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031368104312938814, + "loss": 2.61, + "theoretical_loss": 3.573870158597214, + "tokens_seen": 1250754560 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136710130391174, + "loss": 2.885, + "theoretical_loss": 3.573853067583606, + "tokens_seen": 1250820096 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031366098294884656, + "loss": 2.8004, + "theoretical_loss": 3.5738359777161666, + "tokens_seen": 1250885632 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031365095285857574, + "loss": 2.8157, + "theoretical_loss": 3.573818888994758, + "tokens_seen": 1250951168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136409227683049, + "loss": 2.6856, + "theoretical_loss": 3.5738018014192434, + "tokens_seen": 1251016704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136308926780341, + "loss": 2.625, + "theoretical_loss": 3.573784714989487, + "tokens_seen": 1251082240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136208625877633, + "loss": 2.7976, + "theoretical_loss": 3.5737676297053502, + "tokens_seen": 1251147776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136108324974925, + "loss": 2.5148, + "theoretical_loss": 3.573750545566698, + "tokens_seen": 1251213312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031360080240722165, + "loss": 2.4938, + "theoretical_loss": 3.5737334625733927, + "tokens_seen": 1251278848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135907723169509, + "loss": 2.6541, + "theoretical_loss": 3.5737163807252976, + "tokens_seen": 1251344384 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031358074222668006, + "loss": 2.7878, + "theoretical_loss": 3.5736993000222754, + "tokens_seen": 1251409920 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031357071213640924, + "loss": 2.6138, + "theoretical_loss": 3.5736822204641907, + "tokens_seen": 1251475456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135606820461384, + "loss": 2.6757, + "theoretical_loss": 3.5736651420509054, + "tokens_seen": 1251540992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135506519558676, + "loss": 2.4925, + "theoretical_loss": 3.573648064782284, + "tokens_seen": 1251606528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135406218655968, + "loss": 2.5749, + "theoretical_loss": 3.5736309886581887, + "tokens_seen": 1251672064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 692076, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7197704315185547, + "objective/train/theoretical_loss": 3.573613913678484, + "objective/train/tokens_used": 1272197600, + "theoretical_loss": 3.573613913678484, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313530591775326, + "loss": 2.7437, + "theoretical_loss": 3.573613913678484, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031352056168505515, + "loss": 2.6848, + "theoretical_loss": 3.5735968398430327, + "tokens_seen": 1251803136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135105315947844, + "loss": 2.2709, + "theoretical_loss": 3.5735797671516982, + "tokens_seen": 1251868672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135005015045135, + "loss": 2.3752, + "theoretical_loss": 3.5735626956043443, + "tokens_seen": 1251934208 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031349047141424275, + "loss": 2.6238, + "theoretical_loss": 3.573545625200834, + "tokens_seen": 1251999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031348044132397193, + "loss": 2.3681, + "theoretical_loss": 3.573528555941031, + "tokens_seen": 1252065280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134704112337011, + "loss": 2.7673, + "theoretical_loss": 3.573511487824799, + "tokens_seen": 1252130816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134603811434303, + "loss": 2.812, + "theoretical_loss": 3.5734944208520014, + "tokens_seen": 1252196352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134503510531595, + "loss": 2.654, + "theoretical_loss": 3.5734773550225016, + "tokens_seen": 1252261888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031344032096288865, + "loss": 2.6109, + "theoretical_loss": 3.573460290336163, + "tokens_seen": 1252327424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134302908726179, + "loss": 2.6974, + "theoretical_loss": 3.57344322679285, + "tokens_seen": 1252392960 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313420260782347, + "loss": 2.5873, + "theoretical_loss": 3.5734261643924254, + "tokens_seen": 1252458496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031341023069207625, + "loss": 2.6487, + "theoretical_loss": 3.5734091031347535, + "tokens_seen": 1252524032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031340020060180543, + "loss": 2.6721, + "theoretical_loss": 3.5733920430196973, + "tokens_seen": 1252589568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133901705115346, + "loss": 2.7647, + "theoretical_loss": 3.5733749840471214, + "tokens_seen": 1252655104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133801404212638, + "loss": 2.8827, + "theoretical_loss": 3.5733579262168886, + "tokens_seen": 1252720640 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313370110330993, + "loss": 2.7577, + "theoretical_loss": 3.573340869528863, + "tokens_seen": 1252786176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031336008024072216, + "loss": 2.5466, + "theoretical_loss": 3.573323813982909, + "tokens_seen": 1252851712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133500501504514, + "loss": 2.9619, + "theoretical_loss": 3.573306759578889, + "tokens_seen": 1252917248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133400200601805, + "loss": 2.8034, + "theoretical_loss": 3.573289706316668, + "tokens_seen": 1252982784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031332998996990975, + "loss": 2.592, + "theoretical_loss": 3.57327265419611, + "tokens_seen": 1253048320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133199598796389, + "loss": 2.5597, + "theoretical_loss": 3.5732556032170777, + "tokens_seen": 1253113856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133099297893681, + "loss": 2.5638, + "theoretical_loss": 3.573238553379436, + "tokens_seen": 1253179392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132998996990973, + "loss": 2.7289, + "theoretical_loss": 3.5732215046830484, + "tokens_seen": 1253244928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132898696088265, + "loss": 2.47, + "theoretical_loss": 3.573204457127779, + "tokens_seen": 1253310464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 693502, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.35072660446167, + "objective/train/theoretical_loss": 3.5731874107134916, + "objective/train/tokens_used": 1273836000, + "theoretical_loss": 3.5731874107134916, + "tokens_seen": 1253376000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031327983951855566, + "loss": 2.595, + "theoretical_loss": 3.5731874107134916, + "tokens_seen": 1253376000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132698094282849, + "loss": 2.8433, + "theoretical_loss": 3.57317036544005, + "tokens_seen": 1253441536 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313259779338014, + "loss": 2.8264, + "theoretical_loss": 3.573153321307318, + "tokens_seen": 1253507072 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031324974924774326, + "loss": 2.6044, + "theoretical_loss": 3.5731362783151606, + "tokens_seen": 1253572608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132397191574724, + "loss": 2.6706, + "theoretical_loss": 3.5731192364634414, + "tokens_seen": 1253638144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132296890672016, + "loss": 2.5902, + "theoretical_loss": 3.573102195752024, + "tokens_seen": 1253703680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132196589769308, + "loss": 2.6405, + "theoretical_loss": 3.573085156180773, + "tokens_seen": 1253769216 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031320962888666, + "loss": 2.6236, + "theoretical_loss": 3.5730681177495525, + "tokens_seen": 1253834752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031319959879638916, + "loss": 2.7958, + "theoretical_loss": 3.5730510804582267, + "tokens_seen": 1253900288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031318956870611834, + "loss": 2.6758, + "theoretical_loss": 3.5730340443066595, + "tokens_seen": 1253965824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131795386158475, + "loss": 2.4292, + "theoretical_loss": 3.5730170092947153, + "tokens_seen": 1254031360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031316950852557676, + "loss": 2.8543, + "theoretical_loss": 3.572999975422258, + "tokens_seen": 1254096896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131594784353059, + "loss": 2.7389, + "theoretical_loss": 3.572982942689152, + "tokens_seen": 1254162432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131494483450351, + "loss": 2.5684, + "theoretical_loss": 3.572965911095262, + "tokens_seen": 1254227968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031313941825476425, + "loss": 2.6807, + "theoretical_loss": 3.5729488806404515, + "tokens_seen": 1254293504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131293881644935, + "loss": 2.646, + "theoretical_loss": 3.572931851324585, + "tokens_seen": 1254359040 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031311935807422267, + "loss": 2.7752, + "theoretical_loss": 3.572914823147528, + "tokens_seen": 1254424576 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031310932798395185, + "loss": 2.8694, + "theoretical_loss": 3.5728977961091433, + "tokens_seen": 1254490112 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031309929789368103, + "loss": 2.6834, + "theoretical_loss": 3.5728807702092955, + "tokens_seen": 1254555648 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031308926780341026, + "loss": 2.7661, + "theoretical_loss": 3.57286374544785, + "tokens_seen": 1254621184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130792377131394, + "loss": 2.5441, + "theoretical_loss": 3.5728467218246704, + "tokens_seen": 1254686720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130692076228686, + "loss": 2.7302, + "theoretical_loss": 3.5728296993396214, + "tokens_seen": 1254752256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130591775325978, + "loss": 2.5853, + "theoretical_loss": 3.572812677992567, + "tokens_seen": 1254817792 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313049147442327, + "loss": 2.6742, + "theoretical_loss": 3.572795657783372, + "tokens_seen": 1254883328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130391173520562, + "loss": 2.7661, + "theoretical_loss": 3.5727786387119016, + "tokens_seen": 1254948864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 693948, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6206564903259277, + "objective/train/theoretical_loss": 3.57276162077802, + "objective/train/tokens_used": 1275474400, + "theoretical_loss": 3.57276162077802, + "tokens_seen": 1255014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031302908726178535, + "loss": 2.5951, + "theoretical_loss": 3.57276162077802, + "tokens_seen": 1255014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130190571715146, + "loss": 2.7347, + "theoretical_loss": 3.572744603981591, + "tokens_seen": 1255079936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130090270812437, + "loss": 2.6406, + "theoretical_loss": 3.57272758832248, + "tokens_seen": 1255145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031299899699097295, + "loss": 2.7352, + "theoretical_loss": 3.572710573800551, + "tokens_seen": 1255211008 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031298896690070213, + "loss": 2.5871, + "theoretical_loss": 3.572693560415669, + "tokens_seen": 1255276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129789368104313, + "loss": 2.5636, + "theoretical_loss": 3.5726765481676988, + "tokens_seen": 1255342080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129689067201605, + "loss": 2.6012, + "theoretical_loss": 3.572659537056505, + "tokens_seen": 1255407616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129588766298897, + "loss": 2.8618, + "theoretical_loss": 3.5726425270819515, + "tokens_seen": 1255473152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031294884653961885, + "loss": 2.5406, + "theoretical_loss": 3.572625518243904, + "tokens_seen": 1255538688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129388164493481, + "loss": 2.8062, + "theoretical_loss": 3.572608510542228, + "tokens_seen": 1255604224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129287863590772, + "loss": 2.7838, + "theoretical_loss": 3.572591503976786, + "tokens_seen": 1255669760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031291875626880645, + "loss": 2.6065, + "theoretical_loss": 3.5725744985474446, + "tokens_seen": 1255735296 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031290872617853563, + "loss": 2.6006, + "theoretical_loss": 3.572557494254068, + "tokens_seen": 1255800832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128986960882648, + "loss": 2.5443, + "theoretical_loss": 3.572540491096521, + "tokens_seen": 1255866368 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312888665997994, + "loss": 2.6986, + "theoretical_loss": 3.572523489074668, + "tokens_seen": 1255931904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128786359077232, + "loss": 2.6317, + "theoretical_loss": 3.5725064881883757, + "tokens_seen": 1255997440 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031286860581745236, + "loss": 2.7196, + "theoretical_loss": 3.572489488437507, + "tokens_seen": 1256062976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128585757271816, + "loss": 2.7257, + "theoretical_loss": 3.5724724898219278, + "tokens_seen": 1256128512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128485456369107, + "loss": 2.6395, + "theoretical_loss": 3.5724554923415024, + "tokens_seen": 1256194048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031283851554663995, + "loss": 2.7318, + "theoretical_loss": 3.5724384959960966, + "tokens_seen": 1256259584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128284854563691, + "loss": 2.7062, + "theoretical_loss": 3.572421500785575, + "tokens_seen": 1256325120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128184553660983, + "loss": 2.8028, + "theoretical_loss": 3.5724045067098027, + "tokens_seen": 1256390656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128084252758275, + "loss": 2.8045, + "theoretical_loss": 3.572387513768645, + "tokens_seen": 1256456192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127983951855567, + "loss": 2.6767, + "theoretical_loss": 3.5723705219619664, + "tokens_seen": 1256521728 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031278836509528586, + "loss": 2.5454, + "theoretical_loss": 3.572353531289632, + "tokens_seen": 1256587264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 694968, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5616726875305176, + "objective/train/theoretical_loss": 3.5723365417515076, + "objective/train/tokens_used": 1277112800, + "theoretical_loss": 3.5723365417515076, + "tokens_seen": 1256652800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127783350050151, + "loss": 2.7386, + "theoretical_loss": 3.5723365417515076, + "tokens_seen": 1256652800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127683049147442, + "loss": 2.5781, + "theoretical_loss": 3.5723195533474574, + "tokens_seen": 1256718336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031275827482447346, + "loss": 2.4681, + "theoretical_loss": 3.5723025660773473, + "tokens_seen": 1256783872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127482447342026, + "loss": 2.7769, + "theoretical_loss": 3.5722855799410427, + "tokens_seen": 1256849408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127382146439318, + "loss": 2.6774, + "theoretical_loss": 3.572268594938408, + "tokens_seen": 1256914944 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312728184553661, + "loss": 2.7947, + "theoretical_loss": 3.5722516110693086, + "tokens_seen": 1256980480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127181544633902, + "loss": 2.8695, + "theoretical_loss": 3.572234628333611, + "tokens_seen": 1257046016 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031270812437311936, + "loss": 2.6988, + "theoretical_loss": 3.572217646731178, + "tokens_seen": 1257111552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031269809428284854, + "loss": 2.7323, + "theoretical_loss": 3.5722006662618773, + "tokens_seen": 1257177088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126880641925777, + "loss": 2.5644, + "theoretical_loss": 3.572183686925573, + "tokens_seen": 1257242624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031267803410230696, + "loss": 2.7084, + "theoretical_loss": 3.572166708722131, + "tokens_seen": 1257308160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126680040120361, + "loss": 2.7757, + "theoretical_loss": 3.5721497316514164, + "tokens_seen": 1257373696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126579739217653, + "loss": 2.8024, + "theoretical_loss": 3.5721327557132945, + "tokens_seen": 1257439232 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031264794383149445, + "loss": 2.4816, + "theoretical_loss": 3.572115780907631, + "tokens_seen": 1257504768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126379137412237, + "loss": 2.614, + "theoretical_loss": 3.572098807234291, + "tokens_seen": 1257570304 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031262788365095287, + "loss": 2.636, + "theoretical_loss": 3.5720818346931402, + "tokens_seen": 1257635840 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031261785356068205, + "loss": 2.5206, + "theoretical_loss": 3.572064863284044, + "tokens_seen": 1257701376 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031260782347041123, + "loss": 3.0007, + "theoretical_loss": 3.572047893006868, + "tokens_seen": 1257766912 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031259779338014046, + "loss": 2.7366, + "theoretical_loss": 3.5720309238614774, + "tokens_seen": 1257832448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125877632898696, + "loss": 2.6415, + "theoretical_loss": 3.572013955847738, + "tokens_seen": 1257897984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125777331995988, + "loss": 2.5673, + "theoretical_loss": 3.5719969889655157, + "tokens_seen": 1257963520 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031256770310932795, + "loss": 2.9064, + "theoretical_loss": 3.5719800232146754, + "tokens_seen": 1258029056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125576730190572, + "loss": 2.6507, + "theoretical_loss": 3.5719630585950837, + "tokens_seen": 1258094592 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031254764292878637, + "loss": 3.0106, + "theoretical_loss": 3.5719460951066058, + "tokens_seen": 1258160128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031253761283851555, + "loss": 2.6872, + "theoretical_loss": 3.5719291327491063, + "tokens_seen": 1258225664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 695499, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6464076042175293, + "objective/train/theoretical_loss": 3.5719121715224524, + "objective/train/tokens_used": 1278751200, + "theoretical_loss": 3.5719121715224524, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031252758274824473, + "loss": 2.5867, + "theoretical_loss": 3.5719121715224524, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125175526579739, + "loss": 2.7185, + "theoretical_loss": 3.5718952114265092, + "tokens_seen": 1258356736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125075225677031, + "loss": 2.5542, + "theoretical_loss": 3.5718782524611425, + "tokens_seen": 1258422272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031249749247743233, + "loss": 2.7311, + "theoretical_loss": 3.5718612946262183, + "tokens_seen": 1258487808 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031248746238716146, + "loss": 2.6752, + "theoretical_loss": 3.571844337921602, + "tokens_seen": 1258553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124774322968907, + "loss": 2.6093, + "theoretical_loss": 3.571827382347159, + "tokens_seen": 1258618880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124674022066198, + "loss": 2.5055, + "theoretical_loss": 3.5718104279027565, + "tokens_seen": 1258684416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031245737211634905, + "loss": 2.6449, + "theoretical_loss": 3.571793474588259, + "tokens_seen": 1258749952 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031244734202607823, + "loss": 2.606, + "theoretical_loss": 3.571776522403533, + "tokens_seen": 1258815488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124373119358074, + "loss": 2.6274, + "theoretical_loss": 3.5717595713484442, + "tokens_seen": 1258881024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124272818455366, + "loss": 2.8447, + "theoretical_loss": 3.5717426214228594, + "tokens_seen": 1258946560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031241725175526583, + "loss": 2.7009, + "theoretical_loss": 3.571725672626643, + "tokens_seen": 1259012096 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031240722166499496, + "loss": 2.9124, + "theoretical_loss": 3.5717087249596617, + "tokens_seen": 1259077632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123971915747242, + "loss": 2.7454, + "theoretical_loss": 3.5716917784217816, + "tokens_seen": 1259143168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123871614844533, + "loss": 2.4614, + "theoretical_loss": 3.571674833012869, + "tokens_seen": 1259208704 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031237713139418256, + "loss": 2.8639, + "theoretical_loss": 3.5716578887327897, + "tokens_seen": 1259274240 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031236710130391174, + "loss": 2.7554, + "theoretical_loss": 3.571640945581409, + "tokens_seen": 1259339776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123570712136409, + "loss": 2.7072, + "theoretical_loss": 3.571624003558594, + "tokens_seen": 1259405312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123470411233701, + "loss": 2.4851, + "theoretical_loss": 3.571607062664211, + "tokens_seen": 1259470848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123370110330993, + "loss": 2.5539, + "theoretical_loss": 3.571590122898125, + "tokens_seen": 1259536384 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031232698094282846, + "loss": 2.7576, + "theoretical_loss": 3.5715731842602025, + "tokens_seen": 1259601920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123169508525577, + "loss": 2.5159, + "theoretical_loss": 3.57155624675031, + "tokens_seen": 1259667456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123069207622869, + "loss": 2.6873, + "theoretical_loss": 3.5715393103683137, + "tokens_seen": 1259732992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031229689067201606, + "loss": 2.6168, + "theoretical_loss": 3.57152237511408, + "tokens_seen": 1259798528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122868605817453, + "loss": 2.6619, + "theoretical_loss": 3.571505440987475, + "tokens_seen": 1259864064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 696245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5429320335388184, + "objective/train/theoretical_loss": 3.5714885079883643, + "objective/train/tokens_used": 1280389600, + "theoretical_loss": 3.5714885079883643, + "tokens_seen": 1259929600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122768304914744, + "loss": 2.7015, + "theoretical_loss": 3.5714885079883643, + "tokens_seen": 1259929600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031226680040120366, + "loss": 2.5002, + "theoretical_loss": 3.5714715761166147, + "tokens_seen": 1259995136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122567703109328, + "loss": 2.6852, + "theoretical_loss": 3.571454645372093, + "tokens_seen": 1260060672 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312246740220662, + "loss": 2.5585, + "theoretical_loss": 3.5714377157546644, + "tokens_seen": 1260126208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122367101303912, + "loss": 2.587, + "theoretical_loss": 3.5714207872641963, + "tokens_seen": 1260191744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122266800401204, + "loss": 2.7699, + "theoretical_loss": 3.5714038599005544, + "tokens_seen": 1260257280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031221664994984956, + "loss": 2.8506, + "theoretical_loss": 3.571386933663606, + "tokens_seen": 1260322816 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031220661985957874, + "loss": 2.8147, + "theoretical_loss": 3.5713700085532167, + "tokens_seen": 1260388352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121965897693079, + "loss": 2.7021, + "theoretical_loss": 3.5713530845692527, + "tokens_seen": 1260453888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031218655967903716, + "loss": 2.5115, + "theoretical_loss": 3.5713361617115815, + "tokens_seen": 1260519424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121765295887663, + "loss": 2.513, + "theoretical_loss": 3.5713192399800686, + "tokens_seen": 1260584960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121664994984955, + "loss": 2.5299, + "theoretical_loss": 3.571302319374581, + "tokens_seen": 1260650496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031215646940822465, + "loss": 2.706, + "theoretical_loss": 3.571285399894985, + "tokens_seen": 1260716032 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121464393179539, + "loss": 2.607, + "theoretical_loss": 3.571268481541148, + "tokens_seen": 1260781568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031213640922768307, + "loss": 2.7006, + "theoretical_loss": 3.571251564312936, + "tokens_seen": 1260847104 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031212637913741225, + "loss": 2.7955, + "theoretical_loss": 3.571234648210215, + "tokens_seen": 1260912640 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031211634904714143, + "loss": 2.6337, + "theoretical_loss": 3.5712177332328525, + "tokens_seen": 1260978176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031210631895687066, + "loss": 2.7536, + "theoretical_loss": 3.5712008193807145, + "tokens_seen": 1261043712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120962888665998, + "loss": 2.7754, + "theoretical_loss": 3.571183906653668, + "tokens_seen": 1261109248 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312086258776329, + "loss": 2.5408, + "theoretical_loss": 3.57116699505158, + "tokens_seen": 1261174784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031207622868605815, + "loss": 2.6892, + "theoretical_loss": 3.5711500845743167, + "tokens_seen": 1261240320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120661985957874, + "loss": 2.6009, + "theoretical_loss": 3.5711331752217452, + "tokens_seen": 1261305856 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031205616850551657, + "loss": 2.7901, + "theoretical_loss": 3.571116266993732, + "tokens_seen": 1261371392 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031204613841524575, + "loss": 2.5838, + "theoretical_loss": 3.5710993598901446, + "tokens_seen": 1261436928 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031203610832497493, + "loss": 2.6067, + "theoretical_loss": 3.5710824539108486, + "tokens_seen": 1261502464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 696550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.979339122772217, + "objective/train/theoretical_loss": 3.571065549055712, + "objective/train/tokens_used": 1282028000, + "theoretical_loss": 3.571065549055712, + "tokens_seen": 1261568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120260782347041, + "loss": 2.7769, + "theoretical_loss": 3.571065549055712, + "tokens_seen": 1261568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120160481444333, + "loss": 2.5907, + "theoretical_loss": 3.5710486453246, + "tokens_seen": 1261633536 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031200601805416253, + "loss": 2.6867, + "theoretical_loss": 3.5710317427173814, + "tokens_seen": 1261699072 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031199598796389166, + "loss": 2.6573, + "theoretical_loss": 3.5710148412339224, + "tokens_seen": 1261764608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119859578736209, + "loss": 2.7326, + "theoretical_loss": 3.57099794087409, + "tokens_seen": 1261830144 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031197592778335, + "loss": 2.7819, + "theoretical_loss": 3.5709810416377508, + "tokens_seen": 1261895680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031196589769307925, + "loss": 2.6791, + "theoretical_loss": 3.5709641435247717, + "tokens_seen": 1261961216 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031195586760280843, + "loss": 2.8725, + "theoretical_loss": 3.5709472465350203, + "tokens_seen": 1262026752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119458375125376, + "loss": 2.5824, + "theoretical_loss": 3.5709303506683625, + "tokens_seen": 1262092288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119358074222668, + "loss": 2.5955, + "theoretical_loss": 3.570913455924667, + "tokens_seen": 1262157824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031192577733199603, + "loss": 2.4743, + "theoretical_loss": 3.5708965623037994, + "tokens_seen": 1262223360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031191574724172516, + "loss": 2.6973, + "theoretical_loss": 3.5708796698056275, + "tokens_seen": 1262288896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119057171514544, + "loss": 2.6142, + "theoretical_loss": 3.570862778430018, + "tokens_seen": 1262354432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118956870611835, + "loss": 2.7967, + "theoretical_loss": 3.570845888176839, + "tokens_seen": 1262419968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031188565697091276, + "loss": 2.6092, + "theoretical_loss": 3.5708289990459563, + "tokens_seen": 1262485504 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031187562688064194, + "loss": 2.7344, + "theoretical_loss": 3.570812111037238, + "tokens_seen": 1262551040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118655967903711, + "loss": 2.567, + "theoretical_loss": 3.570795224150551, + "tokens_seen": 1262616576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118555667001003, + "loss": 2.5363, + "theoretical_loss": 3.5707783383857623, + "tokens_seen": 1262682112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118455366098295, + "loss": 2.8668, + "theoretical_loss": 3.5707614537427395, + "tokens_seen": 1262747648 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031183550651955866, + "loss": 2.7614, + "theoretical_loss": 3.5707445702213496, + "tokens_seen": 1262813184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118254764292879, + "loss": 2.6856, + "theoretical_loss": 3.57072768782146, + "tokens_seen": 1262878720 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311815446339017, + "loss": 2.4951, + "theoretical_loss": 3.5707108065429383, + "tokens_seen": 1262944256 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031180541624874626, + "loss": 2.8183, + "theoretical_loss": 3.570693926385651, + "tokens_seen": 1263009792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117953861584754, + "loss": 2.6523, + "theoretical_loss": 3.570677047349467, + "tokens_seen": 1263075328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117853560682046, + "loss": 2.5798, + "theoretical_loss": 3.570660169434252, + "tokens_seen": 1263140864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 697604, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7571280002593994, + "objective/train/theoretical_loss": 3.5706432926398746, + "objective/train/tokens_used": 1283666400, + "theoretical_loss": 3.5706432926398746, + "tokens_seen": 1263206400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117753259779338, + "loss": 2.8312, + "theoretical_loss": 3.5706432926398746, + "tokens_seen": 1263206400 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311765295887663, + "loss": 2.6696, + "theoretical_loss": 3.570626416966201, + "tokens_seen": 1263271936 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031175526579739217, + "loss": 2.6625, + "theoretical_loss": 3.5706095424130995, + "tokens_seen": 1263337472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117452357071214, + "loss": 2.6203, + "theoretical_loss": 3.5705926689804377, + "tokens_seen": 1263403008 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031173520561685053, + "loss": 2.7463, + "theoretical_loss": 3.5705757966680833, + "tokens_seen": 1263468544 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031172517552657976, + "loss": 2.7992, + "theoretical_loss": 3.5705589254759023, + "tokens_seen": 1263534080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117151454363089, + "loss": 2.6173, + "theoretical_loss": 3.570542055403764, + "tokens_seen": 1263599616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117051153460381, + "loss": 2.4746, + "theoretical_loss": 3.570525186451535, + "tokens_seen": 1263665152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116950852557673, + "loss": 2.8257, + "theoretical_loss": 3.5705083186190834, + "tokens_seen": 1263730688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116850551654965, + "loss": 2.5915, + "theoretical_loss": 3.5704914519062756, + "tokens_seen": 1263796224 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031167502507522567, + "loss": 2.6664, + "theoretical_loss": 3.570474586312981, + "tokens_seen": 1263861760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031166499498495485, + "loss": 2.8993, + "theoretical_loss": 3.570457721839066, + "tokens_seen": 1263927296 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031165496489468403, + "loss": 2.4212, + "theoretical_loss": 3.5704408584843987, + "tokens_seen": 1263992832 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031164493480441327, + "loss": 2.7775, + "theoretical_loss": 3.570423996248847, + "tokens_seen": 1264058368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116349047141424, + "loss": 2.809, + "theoretical_loss": 3.570407135132278, + "tokens_seen": 1264123904 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031162487462387163, + "loss": 2.7512, + "theoretical_loss": 3.57039027513456, + "tokens_seen": 1264189440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116148445336008, + "loss": 2.5596, + "theoretical_loss": 3.5703734162555607, + "tokens_seen": 1264254976 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031160481444333, + "loss": 2.6572, + "theoretical_loss": 3.570356558495148, + "tokens_seen": 1264320512 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031159478435305917, + "loss": 2.6785, + "theoretical_loss": 3.570339701853189, + "tokens_seen": 1264386048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031158475426278835, + "loss": 2.6975, + "theoretical_loss": 3.570322846329552, + "tokens_seen": 1264451584 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031157472417251753, + "loss": 2.7055, + "theoretical_loss": 3.570305991924105, + "tokens_seen": 1264517120 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031156469408224677, + "loss": 2.5921, + "theoretical_loss": 3.5702891386367157, + "tokens_seen": 1264582656 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031155466399197595, + "loss": 2.7323, + "theoretical_loss": 3.570272286467252, + "tokens_seen": 1264648192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031154463390170513, + "loss": 2.6489, + "theoretical_loss": 3.5702554354155813, + "tokens_seen": 1264713728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115346038114343, + "loss": 2.6341, + "theoretical_loss": 3.5702385854815724, + "tokens_seen": 1264779264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 698805, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0502676963806152, + "objective/train/theoretical_loss": 3.5702217366650935, + "objective/train/tokens_used": 1285304800, + "theoretical_loss": 3.5702217366650935, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115245737211635, + "loss": 2.8163, + "theoretical_loss": 3.5702217366650935, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031151454363089273, + "loss": 2.5645, + "theoretical_loss": 3.5702048889660114, + "tokens_seen": 1264910336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031150451354062186, + "loss": 2.6297, + "theoretical_loss": 3.5701880423841947, + "tokens_seen": 1264975872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114944834503511, + "loss": 2.7847, + "theoretical_loss": 3.5701711969195116, + "tokens_seen": 1265041408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114844533600802, + "loss": 2.6326, + "theoretical_loss": 3.57015435257183, + "tokens_seen": 1265106944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031147442326980945, + "loss": 2.6998, + "theoretical_loss": 3.570137509341018, + "tokens_seen": 1265172480 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031146439317953863, + "loss": 2.6843, + "theoretical_loss": 3.570120667226943, + "tokens_seen": 1265238016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114543630892678, + "loss": 2.8271, + "theoretical_loss": 3.5701038262294746, + "tokens_seen": 1265303552 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311444332998997, + "loss": 2.7197, + "theoretical_loss": 3.57008698634848, + "tokens_seen": 1265369088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031143430290872623, + "loss": 2.5454, + "theoretical_loss": 3.5700701475838277, + "tokens_seen": 1265434624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031142427281845536, + "loss": 2.8299, + "theoretical_loss": 3.5700533099353855, + "tokens_seen": 1265500160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114142427281846, + "loss": 2.5524, + "theoretical_loss": 3.5700364734030217, + "tokens_seen": 1265565696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114042126379137, + "loss": 2.6633, + "theoretical_loss": 3.5700196379866043, + "tokens_seen": 1265631232 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031139418254764296, + "loss": 2.6066, + "theoretical_loss": 3.5700028036860028, + "tokens_seen": 1265696768 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031138415245737214, + "loss": 2.5845, + "theoretical_loss": 3.569985970501084, + "tokens_seen": 1265762304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113741223671013, + "loss": 2.5936, + "theoretical_loss": 3.5699691384317163, + "tokens_seen": 1265827840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113640922768305, + "loss": 2.4984, + "theoretical_loss": 3.5699523074777693, + "tokens_seen": 1265893376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113540621865597, + "loss": 2.7509, + "theoretical_loss": 3.5699354776391097, + "tokens_seen": 1265958912 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031134403209628886, + "loss": 2.6516, + "theoretical_loss": 3.569918648915607, + "tokens_seen": 1266024448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113340020060181, + "loss": 2.6516, + "theoretical_loss": 3.5699018213071296, + "tokens_seen": 1266089984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113239719157472, + "loss": 2.6194, + "theoretical_loss": 3.5698849948135454, + "tokens_seen": 1266155520 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031131394182547646, + "loss": 2.6923, + "theoretical_loss": 3.5698681694347227, + "tokens_seen": 1266221056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113039117352056, + "loss": 2.6186, + "theoretical_loss": 3.5698513451705307, + "tokens_seen": 1266286592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112938816449348, + "loss": 2.6444, + "theoretical_loss": 3.5698345220208374, + "tokens_seen": 1266352128 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311283851554664, + "loss": 3.0244, + "theoretical_loss": 3.5698176999855114, + "tokens_seen": 1266417664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 699330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7510995864868164, + "objective/train/theoretical_loss": 3.569800879064421, + "objective/train/tokens_used": 1286943200, + "theoretical_loss": 3.569800879064421, + "tokens_seen": 1266483200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112738214643932, + "loss": 2.556, + "theoretical_loss": 3.569800879064421, + "tokens_seen": 1266483200 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031126379137412237, + "loss": 2.7012, + "theoretical_loss": 3.569784059257435, + "tokens_seen": 1266548736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112537612838516, + "loss": 2.6631, + "theoretical_loss": 3.569767240564422, + "tokens_seen": 1266614272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031124373119358073, + "loss": 2.6916, + "theoretical_loss": 3.5697504229852504, + "tokens_seen": 1266679808 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031123370110330996, + "loss": 2.9995, + "theoretical_loss": 3.5697336065197884, + "tokens_seen": 1266745344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112236710130391, + "loss": 2.5175, + "theoretical_loss": 3.569716791167906, + "tokens_seen": 1266810880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112136409227683, + "loss": 2.6245, + "theoretical_loss": 3.56969997692947, + "tokens_seen": 1266876416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112036108324975, + "loss": 2.8514, + "theoretical_loss": 3.569683163804351, + "tokens_seen": 1266941952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111935807422267, + "loss": 2.6541, + "theoretical_loss": 3.5696663517924163, + "tokens_seen": 1267007488 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031118355065195587, + "loss": 2.6908, + "theoretical_loss": 3.5696495408935354, + "tokens_seen": 1267073024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031117352056168505, + "loss": 2.7746, + "theoretical_loss": 3.569632731107576, + "tokens_seen": 1267138560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031116349047141423, + "loss": 2.5903, + "theoretical_loss": 3.569615922434408, + "tokens_seen": 1267204096 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031115346038114347, + "loss": 2.5927, + "theoretical_loss": 3.5695991148739, + "tokens_seen": 1267269632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111434302908726, + "loss": 2.7901, + "theoretical_loss": 3.5695823084259204, + "tokens_seen": 1267335168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031113340020060183, + "loss": 2.7658, + "theoretical_loss": 3.569565503090338, + "tokens_seen": 1267400704 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311123370110331, + "loss": 2.7178, + "theoretical_loss": 3.569548698867022, + "tokens_seen": 1267466240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111133400200602, + "loss": 2.8511, + "theoretical_loss": 3.5695318957558415, + "tokens_seen": 1267531776 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031110330992978937, + "loss": 2.7355, + "theoretical_loss": 3.5695150937566646, + "tokens_seen": 1267597312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031109327983951855, + "loss": 2.7006, + "theoretical_loss": 3.569498292869361, + "tokens_seen": 1267662848 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031108324974924773, + "loss": 2.5922, + "theoretical_loss": 3.5694814930937993, + "tokens_seen": 1267728384 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031107321965897697, + "loss": 2.6327, + "theoretical_loss": 3.5694646944298487, + "tokens_seen": 1267793920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110631895687061, + "loss": 2.594, + "theoretical_loss": 3.569447896877378, + "tokens_seen": 1267859456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031105315947843533, + "loss": 2.5477, + "theoretical_loss": 3.5694311004362556, + "tokens_seen": 1267924992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031104312938816446, + "loss": 2.523, + "theoretical_loss": 3.569414305106352, + "tokens_seen": 1267990528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110330992978937, + "loss": 2.7821, + "theoretical_loss": 3.5693975108875344, + "tokens_seen": 1268056064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 700491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.37016224861145, + "objective/train/theoretical_loss": 3.5693807177796737, + "objective/train/tokens_used": 1288581600, + "theoretical_loss": 3.5693807177796737, + "tokens_seen": 1268121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110230692076229, + "loss": 2.6417, + "theoretical_loss": 3.5693807177796737, + "tokens_seen": 1268121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031101303911735206, + "loss": 2.6757, + "theoretical_loss": 3.5693639257826377, + "tokens_seen": 1268187136 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031100300902708124, + "loss": 2.5802, + "theoretical_loss": 3.5693471348962964, + "tokens_seen": 1268252672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109929789368104, + "loss": 2.6605, + "theoretical_loss": 3.569330345120518, + "tokens_seen": 1268318208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109829488465396, + "loss": 2.6493, + "theoretical_loss": 3.569313556455173, + "tokens_seen": 1268383744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031097291875626883, + "loss": 2.7038, + "theoretical_loss": 3.569296768900129, + "tokens_seen": 1268449280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031096288866599796, + "loss": 2.4932, + "theoretical_loss": 3.5692799824552566, + "tokens_seen": 1268514816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109528585757272, + "loss": 2.8002, + "theoretical_loss": 3.569263197120424, + "tokens_seen": 1268580352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109428284854564, + "loss": 2.7297, + "theoretical_loss": 3.569246412895501, + "tokens_seen": 1268645888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031093279839518556, + "loss": 2.7536, + "theoretical_loss": 3.5692296297803567, + "tokens_seen": 1268711424 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031092276830491474, + "loss": 2.7096, + "theoretical_loss": 3.569212847774861, + "tokens_seen": 1268776960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109127382146439, + "loss": 2.5073, + "theoretical_loss": 3.569196066878882, + "tokens_seen": 1268842496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109027081243731, + "loss": 2.6679, + "theoretical_loss": 3.5691792870922905, + "tokens_seen": 1268908032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031089267803410234, + "loss": 2.5551, + "theoretical_loss": 3.5691625084149545, + "tokens_seen": 1268973568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031088264794383147, + "loss": 2.7831, + "theoretical_loss": 3.5691457308467442, + "tokens_seen": 1269039104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108726178535607, + "loss": 2.7506, + "theoretical_loss": 3.569128954387529, + "tokens_seen": 1269104640 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031086258776328983, + "loss": 2.8059, + "theoretical_loss": 3.5691121790371776, + "tokens_seen": 1269170176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031085255767301906, + "loss": 2.5739, + "theoretical_loss": 3.5690954047955605, + "tokens_seen": 1269235712 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031084252758274824, + "loss": 2.4954, + "theoretical_loss": 3.5690786316625465, + "tokens_seen": 1269301248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108324974924774, + "loss": 2.4753, + "theoretical_loss": 3.5690618596380057, + "tokens_seen": 1269366784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108224674022066, + "loss": 2.819, + "theoretical_loss": 3.569045088721807, + "tokens_seen": 1269432320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108124373119358, + "loss": 2.7176, + "theoretical_loss": 3.5690283189138197, + "tokens_seen": 1269497856 + }, + { + "epoch": 0.38, + "learning_rate": 0.000310802407221665, + "loss": 2.6721, + "theoretical_loss": 3.569011550213914, + "tokens_seen": 1269563392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107923771313942, + "loss": 2.6428, + "theoretical_loss": 3.5689947826219592, + "tokens_seen": 1269628928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107823470411234, + "loss": 2.5535, + "theoretical_loss": 3.5689780161378257, + "tokens_seen": 1269694464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 701131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.939049243927002, + "objective/train/theoretical_loss": 3.568961250761382, + "objective/train/tokens_used": 1290220000, + "theoretical_loss": 3.568961250761382, + "tokens_seen": 1269760000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031077231695085257, + "loss": 2.8504, + "theoretical_loss": 3.568961250761382, + "tokens_seen": 1269760000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107622868605818, + "loss": 2.6175, + "theoretical_loss": 3.568944486492498, + "tokens_seen": 1269825536 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031075225677031093, + "loss": 2.6228, + "theoretical_loss": 3.5689277233310435, + "tokens_seen": 1269891072 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031074222668004016, + "loss": 2.799, + "theoretical_loss": 3.568910961276889, + "tokens_seen": 1269956608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107321965897693, + "loss": 2.7415, + "theoretical_loss": 3.5688942003299027, + "tokens_seen": 1270022144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107221664994985, + "loss": 2.737, + "theoretical_loss": 3.5688774404899553, + "tokens_seen": 1270087680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107121364092277, + "loss": 2.627, + "theoretical_loss": 3.5688606817569166, + "tokens_seen": 1270153216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107021063189569, + "loss": 2.5978, + "theoretical_loss": 3.5688439241306567, + "tokens_seen": 1270218752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031069207622868607, + "loss": 2.9248, + "theoretical_loss": 3.568827167611044, + "tokens_seen": 1270284288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031068204613841525, + "loss": 2.8998, + "theoretical_loss": 3.5688104121979496, + "tokens_seen": 1270349824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031067201604814443, + "loss": 2.6636, + "theoretical_loss": 3.568793657891243, + "tokens_seen": 1270415360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031066198595787367, + "loss": 2.8076, + "theoretical_loss": 3.568776904690794, + "tokens_seen": 1270480896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003106519558676028, + "loss": 2.8176, + "theoretical_loss": 3.5687601525964725, + "tokens_seen": 1270546432 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031064192577733203, + "loss": 2.8466, + "theoretical_loss": 3.5687434016081485, + "tokens_seen": 1270611968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003106318956870612, + "loss": 2.7659, + "theoretical_loss": 3.568726651725692, + "tokens_seen": 1270677504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003106218655967904, + "loss": 2.9059, + "theoretical_loss": 3.5687099029489726, + "tokens_seen": 1270743040 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031061183550651957, + "loss": 2.8367, + "theoretical_loss": 3.5686931552778605, + "tokens_seen": 1270808576 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031060180541624875, + "loss": 2.6191, + "theoretical_loss": 3.5686764087122262, + "tokens_seen": 1270874112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031059177532597793, + "loss": 2.4859, + "theoretical_loss": 3.568659663251939, + "tokens_seen": 1270939648 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031058174523570717, + "loss": 2.7966, + "theoretical_loss": 3.5686429188968694, + "tokens_seen": 1271005184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105717151454363, + "loss": 2.9504, + "theoretical_loss": 3.5686261756468873, + "tokens_seen": 1271070720 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031056168505516553, + "loss": 2.4731, + "theoretical_loss": 3.5686094335018623, + "tokens_seen": 1271136256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031055165496489466, + "loss": 2.5467, + "theoretical_loss": 3.5685926924616655, + "tokens_seen": 1271201792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105416248746239, + "loss": 2.6595, + "theoretical_loss": 3.568575952526167, + "tokens_seen": 1271267328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105315947843531, + "loss": 2.8641, + "theoretical_loss": 3.568559213695236, + "tokens_seen": 1271332864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 702434, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5973994731903076, + "objective/train/theoretical_loss": 3.5685424759687434, + "objective/train/tokens_used": 1291858400, + "theoretical_loss": 3.5685424759687434, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031052156469408226, + "loss": 2.7256, + "theoretical_loss": 3.5685424759687434, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031051153460381144, + "loss": 2.5591, + "theoretical_loss": 3.568525739346559, + "tokens_seen": 1271463936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105015045135406, + "loss": 2.6854, + "theoretical_loss": 3.568509003828553, + "tokens_seen": 1271529472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104914744232698, + "loss": 2.6827, + "theoretical_loss": 3.5684922694145964, + "tokens_seen": 1271595008 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031048144433299904, + "loss": 2.6534, + "theoretical_loss": 3.568475536104559, + "tokens_seen": 1271660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031047141424272816, + "loss": 2.7149, + "theoretical_loss": 3.5684588038983107, + "tokens_seen": 1271726080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104613841524574, + "loss": 2.5994, + "theoretical_loss": 3.5684420727957225, + "tokens_seen": 1271791616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104513540621866, + "loss": 2.7789, + "theoretical_loss": 3.568425342796664, + "tokens_seen": 1271857152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031044132397191576, + "loss": 2.471, + "theoretical_loss": 3.568408613901006, + "tokens_seen": 1271922688 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031043129388164494, + "loss": 2.7984, + "theoretical_loss": 3.5683918861086186, + "tokens_seen": 1271988224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104212637913741, + "loss": 2.7841, + "theoretical_loss": 3.5683751594193733, + "tokens_seen": 1272053760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104112337011033, + "loss": 2.802, + "theoretical_loss": 3.568358433833139, + "tokens_seen": 1272119296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031040120361083254, + "loss": 2.744, + "theoretical_loss": 3.568341709349787, + "tokens_seen": 1272184832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031039117352056167, + "loss": 2.6926, + "theoretical_loss": 3.5683249859691872, + "tokens_seen": 1272250368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103811434302909, + "loss": 2.7375, + "theoretical_loss": 3.568308263691211, + "tokens_seen": 1272315904 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031037111334002003, + "loss": 2.63, + "theoretical_loss": 3.568291542515728, + "tokens_seen": 1272381440 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031036108324974926, + "loss": 2.523, + "theoretical_loss": 3.5682748224426093, + "tokens_seen": 1272446976 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031035105315947844, + "loss": 2.729, + "theoretical_loss": 3.5682581034717256, + "tokens_seen": 1272512512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103410230692076, + "loss": 2.5598, + "theoretical_loss": 3.5682413856029465, + "tokens_seen": 1272578048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103309929789368, + "loss": 2.7564, + "theoretical_loss": 3.5682246688361436, + "tokens_seen": 1272643584 + }, + { + "epoch": 0.39, + "learning_rate": 0.000310320962888666, + "loss": 2.6907, + "theoretical_loss": 3.5682079531711866, + "tokens_seen": 1272709120 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031031093279839517, + "loss": 2.4328, + "theoretical_loss": 3.5681912386079473, + "tokens_seen": 1272774656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103009027081244, + "loss": 2.7671, + "theoretical_loss": 3.568174525146296, + "tokens_seen": 1272840192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031029087261785353, + "loss": 2.6715, + "theoretical_loss": 3.5681578127861027, + "tokens_seen": 1272905728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031028084252758277, + "loss": 2.7638, + "theoretical_loss": 3.5681411015272384, + "tokens_seen": 1272971264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 703530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.578533887863159, + "objective/train/theoretical_loss": 3.568124391369574, + "objective/train/tokens_used": 1293496800, + "theoretical_loss": 3.568124391369574, + "tokens_seen": 1273036800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031027081243731195, + "loss": 2.65, + "theoretical_loss": 3.568124391369574, + "tokens_seen": 1273036800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031026078234704113, + "loss": 2.8089, + "theoretical_loss": 3.5681076823129807, + "tokens_seen": 1273102336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102507522567703, + "loss": 2.5821, + "theoretical_loss": 3.568090974357329, + "tokens_seen": 1273167872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102407221664995, + "loss": 2.7659, + "theoretical_loss": 3.568074267502489, + "tokens_seen": 1273233408 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031023069207622867, + "loss": 2.8061, + "theoretical_loss": 3.568057561748332, + "tokens_seen": 1273298944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102206619859579, + "loss": 2.8214, + "theoretical_loss": 3.5680408570947284, + "tokens_seen": 1273364480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031021063189568703, + "loss": 2.8284, + "theoretical_loss": 3.5680241535415504, + "tokens_seen": 1273430016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031020060180541627, + "loss": 2.9509, + "theoretical_loss": 3.5680074510886675, + "tokens_seen": 1273495552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101905717151454, + "loss": 2.5731, + "theoretical_loss": 3.5679907497359515, + "tokens_seen": 1273561088 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031018054162487463, + "loss": 2.4667, + "theoretical_loss": 3.5679740494832726, + "tokens_seen": 1273626624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101705115346038, + "loss": 2.606, + "theoretical_loss": 3.567957350330502, + "tokens_seen": 1273692160 + }, + { + "epoch": 0.39, + "learning_rate": 0.000310160481444333, + "loss": 2.6988, + "theoretical_loss": 3.5679406522775112, + "tokens_seen": 1273757696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101504513540622, + "loss": 2.7648, + "theoretical_loss": 3.5679239553241704, + "tokens_seen": 1273823232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101404212637914, + "loss": 2.8896, + "theoretical_loss": 3.567907259470351, + "tokens_seen": 1273888768 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031013039117352054, + "loss": 2.6081, + "theoretical_loss": 3.567890564715924, + "tokens_seen": 1273954304 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031012036108324977, + "loss": 2.6571, + "theoretical_loss": 3.5678738710607605, + "tokens_seen": 1274019840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101103309929789, + "loss": 2.8599, + "theoretical_loss": 3.5678571785047315, + "tokens_seen": 1274085376 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031010030090270813, + "loss": 2.8499, + "theoretical_loss": 3.5678404870477083, + "tokens_seen": 1274150912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100902708124373, + "loss": 2.7101, + "theoretical_loss": 3.5678237966895616, + "tokens_seen": 1274216448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100802407221665, + "loss": 2.5664, + "theoretical_loss": 3.5678071074301627, + "tokens_seen": 1274281984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100702106318957, + "loss": 2.9353, + "theoretical_loss": 3.5677904192693832, + "tokens_seen": 1274347520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031006018054162486, + "loss": 2.5256, + "theoretical_loss": 3.567773732207094, + "tokens_seen": 1274413056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100501504513541, + "loss": 2.8448, + "theoretical_loss": 3.567757046243166, + "tokens_seen": 1274478592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100401203610833, + "loss": 2.582, + "theoretical_loss": 3.5677403613774703, + "tokens_seen": 1274544128 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031003009027081246, + "loss": 2.7362, + "theoretical_loss": 3.5677236776098793, + "tokens_seen": 1274609664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 704259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.703928232192993, + "objective/train/theoretical_loss": 3.567706994940263, + "objective/train/tokens_used": 1295135200, + "theoretical_loss": 3.567706994940263, + "tokens_seen": 1274675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031002006018054164, + "loss": 2.5891, + "theoretical_loss": 3.567706994940263, + "tokens_seen": 1274675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100100300902708, + "loss": 2.5855, + "theoretical_loss": 3.567690313368493, + "tokens_seen": 1274740736 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031, + "loss": 2.7842, + "theoretical_loss": 3.5676736328944414, + "tokens_seen": 1274806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030998996990972924, + "loss": 2.7119, + "theoretical_loss": 3.5676569535179787, + "tokens_seen": 1274871808 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030997993981945836, + "loss": 2.541, + "theoretical_loss": 3.5676402752389764, + "tokens_seen": 1274937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099699097291876, + "loss": 2.5326, + "theoretical_loss": 3.567623598057306, + "tokens_seen": 1275002880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099598796389168, + "loss": 2.8544, + "theoretical_loss": 3.5676069219728395, + "tokens_seen": 1275068416 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030994984954864596, + "loss": 2.8376, + "theoretical_loss": 3.5675902469854472, + "tokens_seen": 1275133952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030993981945837514, + "loss": 2.7259, + "theoretical_loss": 3.567573573095001, + "tokens_seen": 1275199488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099297893681043, + "loss": 2.4658, + "theoretical_loss": 3.5675569003013727, + "tokens_seen": 1275265024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099197592778335, + "loss": 2.5857, + "theoretical_loss": 3.5675402286044333, + "tokens_seen": 1275330560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030990972918756274, + "loss": 2.9346, + "theoretical_loss": 3.567523558004055, + "tokens_seen": 1275396096 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030989969909729187, + "loss": 2.5889, + "theoretical_loss": 3.5675068885001084, + "tokens_seen": 1275461632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098896690070211, + "loss": 2.7123, + "theoretical_loss": 3.567490220092466, + "tokens_seen": 1275527168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030987963891675023, + "loss": 2.7163, + "theoretical_loss": 3.5674735527809984, + "tokens_seen": 1275592704 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030986960882647946, + "loss": 2.6532, + "theoretical_loss": 3.5674568865655782, + "tokens_seen": 1275658240 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030985957873620864, + "loss": 2.7162, + "theoretical_loss": 3.567440221446076, + "tokens_seen": 1275723776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098495486459378, + "loss": 2.5943, + "theoretical_loss": 3.5674235574223645, + "tokens_seen": 1275789312 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309839518555667, + "loss": 2.8053, + "theoretical_loss": 3.5674068944943147, + "tokens_seen": 1275854848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098294884653962, + "loss": 2.7737, + "theoretical_loss": 3.5673902326617983, + "tokens_seen": 1275920384 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030981945837512537, + "loss": 2.6332, + "theoretical_loss": 3.567373571924687, + "tokens_seen": 1275985920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098094282848546, + "loss": 2.9085, + "theoretical_loss": 3.567356912282853, + "tokens_seen": 1276051456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030979939819458373, + "loss": 2.4999, + "theoretical_loss": 3.567340253736168, + "tokens_seen": 1276116992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030978936810431297, + "loss": 2.9444, + "theoretical_loss": 3.5673235962845027, + "tokens_seen": 1276182528 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030977933801404215, + "loss": 2.6067, + "theoretical_loss": 3.56730693992773, + "tokens_seen": 1276248064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 705368, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.666820764541626, + "objective/train/theoretical_loss": 3.5672902846657215, + "objective/train/tokens_used": 1296773600, + "theoretical_loss": 3.5672902846657215, + "tokens_seen": 1276313600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030976930792377133, + "loss": 2.7037, + "theoretical_loss": 3.5672902846657215, + "tokens_seen": 1276313600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097592778335005, + "loss": 2.7744, + "theoretical_loss": 3.567273630498349, + "tokens_seen": 1276379136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097492477432297, + "loss": 2.7129, + "theoretical_loss": 3.567256977425484, + "tokens_seen": 1276444672 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030973921765295887, + "loss": 2.5538, + "theoretical_loss": 3.567240325446999, + "tokens_seen": 1276510208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097291875626881, + "loss": 2.8296, + "theoretical_loss": 3.5672236745627655, + "tokens_seen": 1276575744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030971915747241723, + "loss": 2.4411, + "theoretical_loss": 3.5672070247726553, + "tokens_seen": 1276641280 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030970912738214647, + "loss": 2.5156, + "theoretical_loss": 3.5671903760765407, + "tokens_seen": 1276706816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096990972918756, + "loss": 2.86, + "theoretical_loss": 3.5671737284742937, + "tokens_seen": 1276772352 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030968906720160483, + "loss": 2.6477, + "theoretical_loss": 3.567157081965786, + "tokens_seen": 1276837888 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309679037111334, + "loss": 2.4785, + "theoretical_loss": 3.567140436550889, + "tokens_seen": 1276903424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096690070210632, + "loss": 2.4694, + "theoretical_loss": 3.5671237922294763, + "tokens_seen": 1276968960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096589769307924, + "loss": 2.5146, + "theoretical_loss": 3.567107149001419, + "tokens_seen": 1277034496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096489468405216, + "loss": 2.5783, + "theoretical_loss": 3.567090506866589, + "tokens_seen": 1277100032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030963891675025074, + "loss": 2.5938, + "theoretical_loss": 3.5670738658248586, + "tokens_seen": 1277165568 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030962888665997997, + "loss": 2.6706, + "theoretical_loss": 3.5670572258761, + "tokens_seen": 1277231104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096188565697091, + "loss": 2.6656, + "theoretical_loss": 3.567040587020186, + "tokens_seen": 1277296640 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030960882647943833, + "loss": 2.7574, + "theoretical_loss": 3.5670239492569866, + "tokens_seen": 1277362176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095987963891675, + "loss": 2.5667, + "theoretical_loss": 3.5670073125863766, + "tokens_seen": 1277427712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095887662988967, + "loss": 2.6286, + "theoretical_loss": 3.566990677008227, + "tokens_seen": 1277493248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095787362086259, + "loss": 2.6909, + "theoretical_loss": 3.5669740425224097, + "tokens_seen": 1277558784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030956870611835506, + "loss": 2.5731, + "theoretical_loss": 3.566957409128798, + "tokens_seen": 1277624320 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030955867602808424, + "loss": 2.5758, + "theoretical_loss": 3.5669407768272627, + "tokens_seen": 1277689856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095486459378135, + "loss": 2.5629, + "theoretical_loss": 3.566924145617677, + "tokens_seen": 1277755392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095386158475426, + "loss": 2.7665, + "theoretical_loss": 3.5669075154999135, + "tokens_seen": 1277820928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030952858575727184, + "loss": 2.7295, + "theoretical_loss": 3.5668908864738436, + "tokens_seen": 1277886464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 705943, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5287697315216064, + "objective/train/theoretical_loss": 3.5668742585393405, + "objective/train/tokens_used": 1298412000, + "theoretical_loss": 3.5668742585393405, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030951855566700096, + "loss": 2.7983, + "theoretical_loss": 3.5668742585393405, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095085255767302, + "loss": 2.4844, + "theoretical_loss": 3.5668576316962763, + "tokens_seen": 1278017536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094984954864594, + "loss": 2.3433, + "theoretical_loss": 3.5668410059445232, + "tokens_seen": 1278083072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030948846539618856, + "loss": 2.8347, + "theoretical_loss": 3.566824381283954, + "tokens_seen": 1278148608 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030947843530591774, + "loss": 2.5863, + "theoretical_loss": 3.5668077577144404, + "tokens_seen": 1278214144 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309468405215647, + "loss": 2.6227, + "theoretical_loss": 3.5667911352358557, + "tokens_seen": 1278279680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094583751253761, + "loss": 2.7285, + "theoretical_loss": 3.566774513848072, + "tokens_seen": 1278345216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030944834503510534, + "loss": 2.7424, + "theoretical_loss": 3.566757893550962, + "tokens_seen": 1278410752 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030943831494483447, + "loss": 2.6665, + "theoretical_loss": 3.5667412743443982, + "tokens_seen": 1278476288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094282848545637, + "loss": 2.5997, + "theoretical_loss": 3.566724656228253, + "tokens_seen": 1278541824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094182547642929, + "loss": 2.5413, + "theoretical_loss": 3.566708039202399, + "tokens_seen": 1278607360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030940822467402207, + "loss": 2.8092, + "theoretical_loss": 3.5666914232667084, + "tokens_seen": 1278672896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030939819458375125, + "loss": 2.7759, + "theoretical_loss": 3.5666748084210544, + "tokens_seen": 1278738432 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030938816449348043, + "loss": 2.6478, + "theoretical_loss": 3.566658194665309, + "tokens_seen": 1278803968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093781344032096, + "loss": 2.6607, + "theoretical_loss": 3.566641581999346, + "tokens_seen": 1278869504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030936810431293884, + "loss": 2.7618, + "theoretical_loss": 3.566624970423037, + "tokens_seen": 1278935040 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030935807422266797, + "loss": 2.8331, + "theoretical_loss": 3.566608359936255, + "tokens_seen": 1279000576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093480441323972, + "loss": 2.7793, + "theoretical_loss": 3.5665917505388727, + "tokens_seen": 1279066112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030933801404212633, + "loss": 2.6624, + "theoretical_loss": 3.5665751422307634, + "tokens_seen": 1279131648 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030932798395185557, + "loss": 2.6765, + "theoretical_loss": 3.566558535011799, + "tokens_seen": 1279197184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030931795386158475, + "loss": 2.6573, + "theoretical_loss": 3.5665419288818527, + "tokens_seen": 1279262720 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030930792377131393, + "loss": 2.6487, + "theoretical_loss": 3.5665253238407972, + "tokens_seen": 1279328256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030929789368104317, + "loss": 2.7496, + "theoretical_loss": 3.566508719888505, + "tokens_seen": 1279393792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030928786359077235, + "loss": 2.552, + "theoretical_loss": 3.56649211702485, + "tokens_seen": 1279459328 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030927783350050153, + "loss": 2.4734, + "theoretical_loss": 3.5664755152497043, + "tokens_seen": 1279524864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 707215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.725428342819214, + "objective/train/theoretical_loss": 3.566458914562941, + "objective/train/tokens_used": 1300050400, + "theoretical_loss": 3.566458914562941, + "tokens_seen": 1279590400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092678034102307, + "loss": 2.6628, + "theoretical_loss": 3.566458914562941, + "tokens_seen": 1279590400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092577733199599, + "loss": 2.5535, + "theoretical_loss": 3.566442314964432, + "tokens_seen": 1279655936 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030924774322968907, + "loss": 2.4807, + "theoretical_loss": 3.566425716454052, + "tokens_seen": 1279721472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092377131394183, + "loss": 2.7158, + "theoretical_loss": 3.5664091190316727, + "tokens_seen": 1279787008 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030922768304914743, + "loss": 2.5899, + "theoretical_loss": 3.5663925226971678, + "tokens_seen": 1279852544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030921765295887667, + "loss": 2.6971, + "theoretical_loss": 3.5663759274504097, + "tokens_seen": 1279918080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092076228686058, + "loss": 2.5819, + "theoretical_loss": 3.5663593332912713, + "tokens_seen": 1279983616 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030919759277833503, + "loss": 2.5817, + "theoretical_loss": 3.5663427402196266, + "tokens_seen": 1280049152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091875626880642, + "loss": 2.8216, + "theoretical_loss": 3.5663261482353477, + "tokens_seen": 1280114688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091775325977934, + "loss": 2.8327, + "theoretical_loss": 3.5663095573383083, + "tokens_seen": 1280180224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091675025075226, + "loss": 2.63, + "theoretical_loss": 3.5662929675283808, + "tokens_seen": 1280245760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091574724172518, + "loss": 2.6224, + "theoretical_loss": 3.5662763788054392, + "tokens_seen": 1280311296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030914744232698094, + "loss": 2.7127, + "theoretical_loss": 3.5662597911693563, + "tokens_seen": 1280376832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030913741223671017, + "loss": 2.7571, + "theoretical_loss": 3.5662432046200045, + "tokens_seen": 1280442368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091273821464393, + "loss": 2.5277, + "theoretical_loss": 3.5662266191572582, + "tokens_seen": 1280507904 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030911735205616853, + "loss": 2.8114, + "theoretical_loss": 3.56621003478099, + "tokens_seen": 1280573440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091073219658977, + "loss": 2.8727, + "theoretical_loss": 3.566193451491073, + "tokens_seen": 1280638976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090972918756269, + "loss": 2.7229, + "theoretical_loss": 3.566176869287381, + "tokens_seen": 1280704512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090872617853561, + "loss": 2.7898, + "theoretical_loss": 3.5661602881697867, + "tokens_seen": 1280770048 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030907723169508526, + "loss": 2.5655, + "theoretical_loss": 3.5661437081381635, + "tokens_seen": 1280835584 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030906720160481444, + "loss": 2.7232, + "theoretical_loss": 3.5661271291923855, + "tokens_seen": 1280901120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090571715145437, + "loss": 2.7766, + "theoretical_loss": 3.566110551332325, + "tokens_seen": 1280966656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090471414242728, + "loss": 2.7331, + "theoretical_loss": 3.5660939745578553, + "tokens_seen": 1281032192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030903711133400204, + "loss": 2.7573, + "theoretical_loss": 3.5660773988688503, + "tokens_seen": 1281097728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030902708124373116, + "loss": 2.6697, + "theoretical_loss": 3.5660608242651834, + "tokens_seen": 1281163264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 708432, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4614956378936768, + "objective/train/theoretical_loss": 3.566044250746728, + "objective/train/tokens_used": 1301688800, + "theoretical_loss": 3.566044250746728, + "tokens_seen": 1281228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090170511534604, + "loss": 2.6463, + "theoretical_loss": 3.566044250746728, + "tokens_seen": 1281228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090070210631896, + "loss": 2.6089, + "theoretical_loss": 3.5660276783133575, + "tokens_seen": 1281294336 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030899699097291876, + "loss": 2.7834, + "theoretical_loss": 3.566011106964946, + "tokens_seen": 1281359872 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030898696088264794, + "loss": 2.4896, + "theoretical_loss": 3.565994536701365, + "tokens_seen": 1281425408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089769307923772, + "loss": 2.5439, + "theoretical_loss": 3.56597796752249, + "tokens_seen": 1281490944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089669007021063, + "loss": 2.5781, + "theoretical_loss": 3.5659613994281942, + "tokens_seen": 1281556480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030895687061183554, + "loss": 2.6482, + "theoretical_loss": 3.5659448324183503, + "tokens_seen": 1281622016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030894684052156467, + "loss": 2.5109, + "theoretical_loss": 3.5659282664928322, + "tokens_seen": 1281687552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089368104312939, + "loss": 2.7859, + "theoretical_loss": 3.565911701651514, + "tokens_seen": 1281753088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089267803410231, + "loss": 2.5165, + "theoretical_loss": 3.565895137894269, + "tokens_seen": 1281818624 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030891675025075227, + "loss": 2.6697, + "theoretical_loss": 3.565878575220971, + "tokens_seen": 1281884160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030890672016048145, + "loss": 2.5868, + "theoretical_loss": 3.565862013631493, + "tokens_seen": 1281949696 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030889669007021063, + "loss": 2.6787, + "theoretical_loss": 3.5658454531257093, + "tokens_seen": 1282015232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088866599799398, + "loss": 2.6519, + "theoretical_loss": 3.565828893703493, + "tokens_seen": 1282080768 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030887662988966904, + "loss": 2.785, + "theoretical_loss": 3.5658123353647193, + "tokens_seen": 1282146304 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030886659979939817, + "loss": 2.6485, + "theoretical_loss": 3.56579577810926, + "tokens_seen": 1282211840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088565697091274, + "loss": 2.519, + "theoretical_loss": 3.56577922193699, + "tokens_seen": 1282277376 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030884653961885653, + "loss": 2.7086, + "theoretical_loss": 3.5657626668477826, + "tokens_seen": 1282342912 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030883650952858577, + "loss": 2.7554, + "theoretical_loss": 3.5657461128415124, + "tokens_seen": 1282408448 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030882647943831495, + "loss": 2.4995, + "theoretical_loss": 3.5657295599180525, + "tokens_seen": 1282473984 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030881644934804413, + "loss": 2.4584, + "theoretical_loss": 3.5657130080772763, + "tokens_seen": 1282539520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088064192577733, + "loss": 2.6642, + "theoretical_loss": 3.565696457319059, + "tokens_seen": 1282605056 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030879638916750255, + "loss": 2.6031, + "theoretical_loss": 3.5656799076432737, + "tokens_seen": 1282670592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087863590772317, + "loss": 2.5193, + "theoretical_loss": 3.565663359049794, + "tokens_seen": 1282736128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087763289869609, + "loss": 2.488, + "theoretical_loss": 3.5656468115384943, + "tokens_seen": 1282801664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 708993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5532875061035156, + "objective/train/theoretical_loss": 3.565630265109249, + "objective/train/tokens_used": 1303327200, + "theoretical_loss": 3.565630265109249, + "tokens_seen": 1282867200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030876629889669004, + "loss": 2.6164, + "theoretical_loss": 3.565630265109249, + "tokens_seen": 1282867200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030875626880641927, + "loss": 2.2776, + "theoretical_loss": 3.565613719761931, + "tokens_seen": 1282932736 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030874623871614845, + "loss": 2.7188, + "theoretical_loss": 3.565597175496414, + "tokens_seen": 1282998272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030873620862587763, + "loss": 2.7206, + "theoretical_loss": 3.565580632312574, + "tokens_seen": 1283063808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087261785356068, + "loss": 2.6497, + "theoretical_loss": 3.5655640902102834, + "tokens_seen": 1283129344 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308716148445336, + "loss": 2.8397, + "theoretical_loss": 3.565547549189417, + "tokens_seen": 1283194880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087061183550652, + "loss": 2.5893, + "theoretical_loss": 3.565531009249848, + "tokens_seen": 1283260416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086960882647944, + "loss": 2.5696, + "theoretical_loss": 3.565514470391452, + "tokens_seen": 1283325952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030868605817452354, + "loss": 2.4794, + "theoretical_loss": 3.5654979326141016, + "tokens_seen": 1283391488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086760280842528, + "loss": 2.5306, + "theoretical_loss": 3.5654813959176717, + "tokens_seen": 1283457024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086659979939819, + "loss": 2.5971, + "theoretical_loss": 3.5654648603020362, + "tokens_seen": 1283522560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030865596790371114, + "loss": 2.49, + "theoretical_loss": 3.5654483257670693, + "tokens_seen": 1283588096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086459378134403, + "loss": 2.5352, + "theoretical_loss": 3.5654317923126455, + "tokens_seen": 1283653632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086359077231695, + "loss": 2.4462, + "theoretical_loss": 3.565415259938639, + "tokens_seen": 1283719168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086258776328987, + "loss": 2.6472, + "theoretical_loss": 3.565398728644923, + "tokens_seen": 1283784704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086158475426279, + "loss": 2.6344, + "theoretical_loss": 3.565382198431374, + "tokens_seen": 1283850240 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030860581745235704, + "loss": 2.6027, + "theoretical_loss": 3.565365669297864, + "tokens_seen": 1283915776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085957873620863, + "loss": 2.706, + "theoretical_loss": 3.565349141244268, + "tokens_seen": 1283981312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085857572718154, + "loss": 2.759, + "theoretical_loss": 3.5653326142704618, + "tokens_seen": 1284046848 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030857572718154464, + "loss": 2.7042, + "theoretical_loss": 3.565316088376317, + "tokens_seen": 1284112384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085656970912738, + "loss": 2.8362, + "theoretical_loss": 3.5652995635617106, + "tokens_seen": 1284177920 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308555667001003, + "loss": 2.7046, + "theoretical_loss": 3.5652830398265154, + "tokens_seen": 1284243456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030854563691073224, + "loss": 2.7368, + "theoretical_loss": 3.5652665171706066, + "tokens_seen": 1284308992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030853560682046137, + "loss": 2.4158, + "theoretical_loss": 3.565249995593858, + "tokens_seen": 1284374528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085255767301906, + "loss": 2.6283, + "theoretical_loss": 3.5652334750961447, + "tokens_seen": 1284440064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 710075, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5692903995513916, + "objective/train/theoretical_loss": 3.5652169556773403, + "objective/train/tokens_used": 1304965600, + "theoretical_loss": 3.5652169556773403, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085155466399198, + "loss": 2.6864, + "theoretical_loss": 3.5652169556773403, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030850551654964896, + "loss": 2.8175, + "theoretical_loss": 3.56520043733732, + "tokens_seen": 1284571136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030849548645937814, + "loss": 2.7832, + "theoretical_loss": 3.5651839200759587, + "tokens_seen": 1284636672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084854563691074, + "loss": 2.6762, + "theoretical_loss": 3.5651674038931302, + "tokens_seen": 1284702208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084754262788365, + "loss": 2.5926, + "theoretical_loss": 3.5651508887887093, + "tokens_seen": 1284767744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030846539618856574, + "loss": 2.8683, + "theoretical_loss": 3.5651343747625703, + "tokens_seen": 1284833280 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030845536609829487, + "loss": 2.6109, + "theoretical_loss": 3.5651178618145885, + "tokens_seen": 1284898816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084453360080241, + "loss": 2.4738, + "theoretical_loss": 3.565101349944638, + "tokens_seen": 1284964352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084353059177533, + "loss": 2.624, + "theoretical_loss": 3.5650848391525933, + "tokens_seen": 1285029888 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030842527582748247, + "loss": 2.631, + "theoretical_loss": 3.565068329438329, + "tokens_seen": 1285095424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030841524573721165, + "loss": 2.7388, + "theoretical_loss": 3.565051820801721, + "tokens_seen": 1285160960 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030840521564694083, + "loss": 2.6277, + "theoretical_loss": 3.5650353132426424, + "tokens_seen": 1285226496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030839518555667, + "loss": 2.6851, + "theoretical_loss": 3.5650188067609685, + "tokens_seen": 1285292032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030838515546639924, + "loss": 2.6007, + "theoretical_loss": 3.5650023013565746, + "tokens_seen": 1285357568 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030837512537612837, + "loss": 2.5723, + "theoretical_loss": 3.564985797029335, + "tokens_seen": 1285423104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083650952858576, + "loss": 2.4376, + "theoretical_loss": 3.5649692937791246, + "tokens_seen": 1285488640 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030835506519558673, + "loss": 2.6476, + "theoretical_loss": 3.564952791605818, + "tokens_seen": 1285554176 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030834503510531597, + "loss": 2.5162, + "theoretical_loss": 3.56493629050929, + "tokens_seen": 1285619712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030833500501504515, + "loss": 2.7669, + "theoretical_loss": 3.564919790489416, + "tokens_seen": 1285685248 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030832497492477433, + "loss": 2.7172, + "theoretical_loss": 3.56490329154607, + "tokens_seen": 1285750784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083149448345035, + "loss": 2.6172, + "theoretical_loss": 3.564886793679128, + "tokens_seen": 1285816320 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030830491474423275, + "loss": 2.6885, + "theoretical_loss": 3.564870296888464, + "tokens_seen": 1285881856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082948846539619, + "loss": 2.6611, + "theoretical_loss": 3.5648538011739537, + "tokens_seen": 1285947392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082848545636911, + "loss": 2.6482, + "theoretical_loss": 3.5648373065354715, + "tokens_seen": 1286012928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030827482447342024, + "loss": 2.8201, + "theoretical_loss": 3.564820812972892, + "tokens_seen": 1286078464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 710665, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8468892574310303, + "objective/train/theoretical_loss": 3.5648043204860915, + "objective/train/tokens_used": 1306604000, + "theoretical_loss": 3.5648043204860915, + "tokens_seen": 1286144000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030826479438314947, + "loss": 2.7577, + "theoretical_loss": 3.5648043204860915, + "tokens_seen": 1286144000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030825476429287865, + "loss": 2.6774, + "theoretical_loss": 3.5647878290749433, + "tokens_seen": 1286209536 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030824473420260783, + "loss": 2.4722, + "theoretical_loss": 3.564771338739324, + "tokens_seen": 1286275072 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308234704112337, + "loss": 2.7426, + "theoretical_loss": 3.564754849479108, + "tokens_seen": 1286340608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082246740220662, + "loss": 2.6997, + "theoretical_loss": 3.56473836129417, + "tokens_seen": 1286406144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082146439317954, + "loss": 2.5188, + "theoretical_loss": 3.564721874184386, + "tokens_seen": 1286471680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082046138415246, + "loss": 2.6416, + "theoretical_loss": 3.5647053881496307, + "tokens_seen": 1286537216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030819458375125374, + "loss": 2.6886, + "theoretical_loss": 3.564688903189779, + "tokens_seen": 1286602752 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308184553660983, + "loss": 2.8147, + "theoretical_loss": 3.5646724193047064, + "tokens_seen": 1286668288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081745235707121, + "loss": 2.6382, + "theoretical_loss": 3.5646559364942876, + "tokens_seen": 1286733824 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030816449348044134, + "loss": 2.6757, + "theoretical_loss": 3.5646394547583986, + "tokens_seen": 1286799360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081544633901705, + "loss": 2.8823, + "theoretical_loss": 3.5646229740969138, + "tokens_seen": 1286864896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081444332998997, + "loss": 2.6111, + "theoretical_loss": 3.564606494509709, + "tokens_seen": 1286930432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081344032096289, + "loss": 2.7325, + "theoretical_loss": 3.5645900159966595, + "tokens_seen": 1286995968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081243731193581, + "loss": 2.8163, + "theoretical_loss": 3.5645735385576405, + "tokens_seen": 1287061504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030811434302908724, + "loss": 2.6988, + "theoretical_loss": 3.5645570621925264, + "tokens_seen": 1287127040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081043129388165, + "loss": 2.5156, + "theoretical_loss": 3.5645405869011944, + "tokens_seen": 1287192576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080942828485456, + "loss": 2.4571, + "theoretical_loss": 3.5645241126835185, + "tokens_seen": 1287258112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030808425275827484, + "loss": 2.6575, + "theoretical_loss": 3.564507639539374, + "tokens_seen": 1287323648 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308074222668004, + "loss": 2.7198, + "theoretical_loss": 3.564491167468637, + "tokens_seen": 1287389184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080641925777332, + "loss": 2.6747, + "theoretical_loss": 3.5644746964711826, + "tokens_seen": 1287454720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080541624874624, + "loss": 2.4686, + "theoretical_loss": 3.5644582265468863, + "tokens_seen": 1287520256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030804413239719157, + "loss": 2.5834, + "theoretical_loss": 3.5644417576956235, + "tokens_seen": 1287585792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030803410230692075, + "loss": 2.755, + "theoretical_loss": 3.5644252899172697, + "tokens_seen": 1287651328 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030802407221665, + "loss": 2.8153, + "theoretical_loss": 3.564408823211701, + "tokens_seen": 1287716864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 711678, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.727043628692627, + "objective/train/theoretical_loss": 3.5643923575787912, + "objective/train/tokens_used": 1308242400, + "theoretical_loss": 3.5643923575787912, + "tokens_seen": 1287782400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080140421263791, + "loss": 2.6927, + "theoretical_loss": 3.5643923575787912, + "tokens_seen": 1287782400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030800401203610834, + "loss": 2.5504, + "theoretical_loss": 3.5643758930184175, + "tokens_seen": 1287847936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079939819458375, + "loss": 2.5323, + "theoretical_loss": 3.564359429530455, + "tokens_seen": 1287913472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079839518555667, + "loss": 2.6744, + "theoretical_loss": 3.564342967114779, + "tokens_seen": 1287979008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079739217652959, + "loss": 2.4857, + "theoretical_loss": 3.564326505771265, + "tokens_seen": 1288044544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030796389167502507, + "loss": 2.5586, + "theoretical_loss": 3.5643100454997896, + "tokens_seen": 1288110080 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030795386158475425, + "loss": 2.8358, + "theoretical_loss": 3.564293586300227, + "tokens_seen": 1288175616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079438314944835, + "loss": 2.7387, + "theoretical_loss": 3.5642771281724546, + "tokens_seen": 1288241152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079338014042126, + "loss": 2.7444, + "theoretical_loss": 3.5642606711163465, + "tokens_seen": 1288306688 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030792377131394185, + "loss": 2.7622, + "theoretical_loss": 3.5642442151317795, + "tokens_seen": 1288372224 + }, + { + "epoch": 0.39, + "learning_rate": 0.000307913741223671, + "loss": 2.9028, + "theoretical_loss": 3.5642277602186283, + "tokens_seen": 1288437760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079037111334002, + "loss": 2.5815, + "theoretical_loss": 3.5642113063767695, + "tokens_seen": 1288503296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078936810431294, + "loss": 2.6371, + "theoretical_loss": 3.564194853606079, + "tokens_seen": 1288568832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030788365095285857, + "loss": 2.5217, + "theoretical_loss": 3.5641784019064318, + "tokens_seen": 1288634368 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030787362086258775, + "loss": 2.7574, + "theoretical_loss": 3.564161951277704, + "tokens_seen": 1288699904 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030786359077231693, + "loss": 2.6885, + "theoretical_loss": 3.564145501719772, + "tokens_seen": 1288765440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078535606820461, + "loss": 2.4649, + "theoretical_loss": 3.5641290532325107, + "tokens_seen": 1288830976 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030784353059177535, + "loss": 2.7864, + "theoretical_loss": 3.5641126058157964, + "tokens_seen": 1288896512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078335005015045, + "loss": 2.7527, + "theoretical_loss": 3.5640961594695053, + "tokens_seen": 1288962048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078234704112337, + "loss": 2.3971, + "theoretical_loss": 3.564079714193513, + "tokens_seen": 1289027584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078134403209629, + "loss": 2.8447, + "theoretical_loss": 3.5640632699876957, + "tokens_seen": 1289093120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078034102306921, + "loss": 2.5957, + "theoretical_loss": 3.564046826851929, + "tokens_seen": 1289158656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077933801404213, + "loss": 2.8842, + "theoretical_loss": 3.564030384786089, + "tokens_seen": 1289224192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030778335005015044, + "loss": 2.5096, + "theoretical_loss": 3.564013943790052, + "tokens_seen": 1289289728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030777331995987967, + "loss": 2.8288, + "theoretical_loss": 3.5639975038636935, + "tokens_seen": 1289355264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 712324, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.687561273574829, + "objective/train/theoretical_loss": 3.5639810650068897, + "objective/train/tokens_used": 1309880800, + "theoretical_loss": 3.5639810650068897, + "tokens_seen": 1289420800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030776328986960885, + "loss": 2.7162, + "theoretical_loss": 3.5639810650068897, + "tokens_seen": 1289420800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030775325977933803, + "loss": 2.6648, + "theoretical_loss": 3.563964627219517, + "tokens_seen": 1289486336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077432296890672, + "loss": 2.4395, + "theoretical_loss": 3.5639481905014514, + "tokens_seen": 1289551872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077331995987964, + "loss": 2.6302, + "theoretical_loss": 3.5639317548525686, + "tokens_seen": 1289617408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077231695085256, + "loss": 2.6502, + "theoretical_loss": 3.563915320272745, + "tokens_seen": 1289682944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077131394182548, + "loss": 2.604, + "theoretical_loss": 3.563898886761857, + "tokens_seen": 1289748480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030770310932798394, + "loss": 2.6581, + "theoretical_loss": 3.56388245431978, + "tokens_seen": 1289814016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076930792377132, + "loss": 2.9133, + "theoretical_loss": 3.563866022946391, + "tokens_seen": 1289879552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076830491474423, + "loss": 2.7273, + "theoretical_loss": 3.5638495926415654, + "tokens_seen": 1289945088 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030767301905717154, + "loss": 2.6406, + "theoretical_loss": 3.5638331634051807, + "tokens_seen": 1290010624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076629889669007, + "loss": 2.7489, + "theoretical_loss": 3.5638167352371117, + "tokens_seen": 1290076160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076529588766299, + "loss": 2.6471, + "theoretical_loss": 3.5638003081372354, + "tokens_seen": 1290141696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076429287863591, + "loss": 2.8173, + "theoretical_loss": 3.563783882105428, + "tokens_seen": 1290207232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076328986960883, + "loss": 2.789, + "theoretical_loss": 3.563767457141566, + "tokens_seen": 1290272768 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030762286860581744, + "loss": 2.732, + "theoretical_loss": 3.5637510332455253, + "tokens_seen": 1290338304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076128385155467, + "loss": 2.7352, + "theoretical_loss": 3.5637346104171828, + "tokens_seen": 1290403840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076028084252758, + "loss": 2.6191, + "theoretical_loss": 3.5637181886564147, + "tokens_seen": 1290469376 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030759277833500504, + "loss": 2.5478, + "theoretical_loss": 3.5637017679630967, + "tokens_seen": 1290534912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075827482447342, + "loss": 2.5346, + "theoretical_loss": 3.563685348337106, + "tokens_seen": 1290600448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075727181544634, + "loss": 2.6512, + "theoretical_loss": 3.5636689297783186, + "tokens_seen": 1290665984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075626880641926, + "loss": 2.564, + "theoretical_loss": 3.5636525122866116, + "tokens_seen": 1290731520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030755265797392177, + "loss": 2.4989, + "theoretical_loss": 3.5636360958618605, + "tokens_seen": 1290797056 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030754262788365095, + "loss": 2.745, + "theoretical_loss": 3.563619680503942, + "tokens_seen": 1290862592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075325977933802, + "loss": 2.8103, + "theoretical_loss": 3.563603266212734, + "tokens_seen": 1290928128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075225677031093, + "loss": 2.5077, + "theoretical_loss": 3.563586852988111, + "tokens_seen": 1290993664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 713206, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9165244102478027, + "objective/train/theoretical_loss": 3.563570440829951, + "objective/train/tokens_used": 1311519200, + "theoretical_loss": 3.563570440829951, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030751253761283854, + "loss": 2.7767, + "theoretical_loss": 3.563570440829951, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075025075225677, + "loss": 2.5896, + "theoretical_loss": 3.56355402973813, + "tokens_seen": 1291124736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074924774322969, + "loss": 2.5465, + "theoretical_loss": 3.5635376197125246, + "tokens_seen": 1291190272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074824473420261, + "loss": 2.5913, + "theoretical_loss": 3.5635212107530116, + "tokens_seen": 1291255808 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030747241725175527, + "loss": 2.6442, + "theoretical_loss": 3.563504802859468, + "tokens_seen": 1291321344 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030746238716148445, + "loss": 2.7839, + "theoretical_loss": 3.5634883960317687, + "tokens_seen": 1291386880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074523570712137, + "loss": 2.7687, + "theoretical_loss": 3.5634719902697927, + "tokens_seen": 1291452416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074423269809428, + "loss": 2.8589, + "theoretical_loss": 3.563455585573416, + "tokens_seen": 1291517952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030743229689067205, + "loss": 2.8238, + "theoretical_loss": 3.563439181942514, + "tokens_seen": 1291583488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074222668004012, + "loss": 2.6259, + "theoretical_loss": 3.563422779376965, + "tokens_seen": 1291649024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074122367101304, + "loss": 2.5173, + "theoretical_loss": 3.563406377876645, + "tokens_seen": 1291714560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074022066198596, + "loss": 2.7689, + "theoretical_loss": 3.563389977441431, + "tokens_seen": 1291780096 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030739217652958877, + "loss": 2.5174, + "theoretical_loss": 3.5633735780711997, + "tokens_seen": 1291845632 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030738214643931795, + "loss": 2.3589, + "theoretical_loss": 3.563357179765828, + "tokens_seen": 1291911168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030737211634904713, + "loss": 2.7557, + "theoretical_loss": 3.563340782525193, + "tokens_seen": 1291976704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073620862587763, + "loss": 2.6461, + "theoretical_loss": 3.563324386349171, + "tokens_seen": 1292042240 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030735205616850555, + "loss": 2.5942, + "theoretical_loss": 3.5633079912376395, + "tokens_seen": 1292107776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073420260782347, + "loss": 2.5485, + "theoretical_loss": 3.563291597190475, + "tokens_seen": 1292173312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073319959879639, + "loss": 2.8998, + "theoretical_loss": 3.5632752042075544, + "tokens_seen": 1292238848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073219658976931, + "loss": 2.4768, + "theoretical_loss": 3.5632588122887547, + "tokens_seen": 1292304384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073119358074223, + "loss": 2.6437, + "theoretical_loss": 3.5632424214339533, + "tokens_seen": 1292369920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030730190571715146, + "loss": 2.5855, + "theoretical_loss": 3.563226031643026, + "tokens_seen": 1292435456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030729187562688064, + "loss": 2.9889, + "theoretical_loss": 3.5632096429158517, + "tokens_seen": 1292500992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072818455366098, + "loss": 2.6217, + "theoretical_loss": 3.5631932552523056, + "tokens_seen": 1292566528 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030727181544633905, + "loss": 2.6415, + "theoretical_loss": 3.5631768686522656, + "tokens_seen": 1292632064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 713898, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8086705207824707, + "objective/train/theoretical_loss": 3.5631604831156087, + "objective/train/tokens_used": 1313157600, + "theoretical_loss": 3.5631604831156087, + "tokens_seen": 1292697600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072617853560682, + "loss": 2.9427, + "theoretical_loss": 3.5631604831156087, + "tokens_seen": 1292697600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072517552657974, + "loss": 2.5816, + "theoretical_loss": 3.563144098642212, + "tokens_seen": 1292763136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030724172517552654, + "loss": 2.5896, + "theoretical_loss": 3.5631277152319525, + "tokens_seen": 1292828672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072316950852558, + "loss": 2.4494, + "theoretical_loss": 3.5631113328847075, + "tokens_seen": 1292894208 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030722166499498496, + "loss": 2.7717, + "theoretical_loss": 3.563094951600354, + "tokens_seen": 1292959744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030721163490471414, + "loss": 2.4096, + "theoretical_loss": 3.563078571378769, + "tokens_seen": 1293025280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072016048144433, + "loss": 2.6803, + "theoretical_loss": 3.5630621922198302, + "tokens_seen": 1293090816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071915747241725, + "loss": 2.7954, + "theoretical_loss": 3.5630458141234147, + "tokens_seen": 1293156352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071815446339017, + "loss": 2.4529, + "theoretical_loss": 3.563029437089399, + "tokens_seen": 1293221888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071715145436309, + "loss": 2.5035, + "theoretical_loss": 3.5630130611176614, + "tokens_seen": 1293287424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030716148445336005, + "loss": 2.6606, + "theoretical_loss": 3.562996686208078, + "tokens_seen": 1293352960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071514543630893, + "loss": 2.4751, + "theoretical_loss": 3.5629803123605273, + "tokens_seen": 1293418496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030714142427281846, + "loss": 2.621, + "theoretical_loss": 3.562963939574886, + "tokens_seen": 1293484032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030713139418254764, + "loss": 2.6346, + "theoretical_loss": 3.5629475678510314, + "tokens_seen": 1293549568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071213640922768, + "loss": 2.5764, + "theoretical_loss": 3.562931197188841, + "tokens_seen": 1293615104 + }, + { + "epoch": 0.39, + "learning_rate": 0.000307111334002006, + "loss": 2.8367, + "theoretical_loss": 3.562914827588192, + "tokens_seen": 1293680640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071013039117352, + "loss": 2.569, + "theoretical_loss": 3.562898459048962, + "tokens_seen": 1293746176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070912738214644, + "loss": 2.8707, + "theoretical_loss": 3.562882091571028, + "tokens_seen": 1293811712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030708124373119355, + "loss": 2.5853, + "theoretical_loss": 3.5628657251542686, + "tokens_seen": 1293877248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070712136409228, + "loss": 2.5337, + "theoretical_loss": 3.5628493597985598, + "tokens_seen": 1293942784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070611835506519, + "loss": 2.8478, + "theoretical_loss": 3.56283299550378, + "tokens_seen": 1294008320 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030705115346038115, + "loss": 2.5145, + "theoretical_loss": 3.562816632269806, + "tokens_seen": 1294073856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070411233701104, + "loss": 2.727, + "theoretical_loss": 3.562800270096516, + "tokens_seen": 1294139392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070310932798395, + "loss": 2.8267, + "theoretical_loss": 3.5627839089837874, + "tokens_seen": 1294204928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030702106318956874, + "loss": 2.6774, + "theoretical_loss": 3.5627675489314976, + "tokens_seen": 1294270464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 715089, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8689498901367188, + "objective/train/theoretical_loss": 3.562751189939524, + "objective/train/tokens_used": 1314796000, + "theoretical_loss": 3.562751189939524, + "tokens_seen": 1294336000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070110330992979, + "loss": 2.6044, + "theoretical_loss": 3.562751189939524, + "tokens_seen": 1294336000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070010030090271, + "loss": 2.5002, + "theoretical_loss": 3.5627348320077443, + "tokens_seen": 1294401536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069909729187563, + "loss": 2.6302, + "theoretical_loss": 3.5627184751360366, + "tokens_seen": 1294467072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030698094282848547, + "loss": 2.6883, + "theoretical_loss": 3.5627021193242774, + "tokens_seen": 1294532608 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030697091273821465, + "loss": 2.6914, + "theoretical_loss": 3.562685764572346, + "tokens_seen": 1294598144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069608826479439, + "loss": 2.6019, + "theoretical_loss": 3.5626694108801185, + "tokens_seen": 1294663680 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306950852557673, + "loss": 2.8642, + "theoretical_loss": 3.5626530582474736, + "tokens_seen": 1294729216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030694082246740225, + "loss": 2.8976, + "theoretical_loss": 3.5626367066742883, + "tokens_seen": 1294794752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069307923771314, + "loss": 2.6636, + "theoretical_loss": 3.5626203561604415, + "tokens_seen": 1294860288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069207622868606, + "loss": 2.6443, + "theoretical_loss": 3.5626040067058096, + "tokens_seen": 1294925824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069107321965898, + "loss": 2.8291, + "theoretical_loss": 3.562587658310271, + "tokens_seen": 1294991360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030690070210631897, + "loss": 2.6902, + "theoretical_loss": 3.562571310973704, + "tokens_seen": 1295056896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030689067201604815, + "loss": 2.6235, + "theoretical_loss": 3.562554964695985, + "tokens_seen": 1295122432 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030688064192577733, + "loss": 2.5321, + "theoretical_loss": 3.5625386194769932, + "tokens_seen": 1295187968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068706118355065, + "loss": 2.5641, + "theoretical_loss": 3.5625222753166064, + "tokens_seen": 1295253504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030686058174523575, + "loss": 2.625, + "theoretical_loss": 3.5625059322147017, + "tokens_seen": 1295319040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068505516549649, + "loss": 2.8001, + "theoretical_loss": 3.5624895901711575, + "tokens_seen": 1295384576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068405215646941, + "loss": 2.7228, + "theoretical_loss": 3.5624732491858513, + "tokens_seen": 1295450112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068304914744233, + "loss": 2.7979, + "theoretical_loss": 3.5624569092586618, + "tokens_seen": 1295515648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068204613841525, + "loss": 2.6856, + "theoretical_loss": 3.5624405703894664, + "tokens_seen": 1295581184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030681043129388166, + "loss": 2.581, + "theoretical_loss": 3.562424232578143, + "tokens_seen": 1295646720 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030680040120361084, + "loss": 2.7789, + "theoretical_loss": 3.56240789582457, + "tokens_seen": 1295712256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030679037111334, + "loss": 2.7325, + "theoretical_loss": 3.562391560128625, + "tokens_seen": 1295777792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030678034102306925, + "loss": 2.6629, + "theoretical_loss": 3.562375225490187, + "tokens_seen": 1295843328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067703109327984, + "loss": 2.5312, + "theoretical_loss": 3.562358891909132, + "tokens_seen": 1295908864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 716367, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.913911819458008, + "objective/train/theoretical_loss": 3.5623425593853404, + "objective/train/tokens_used": 1316434400, + "theoretical_loss": 3.5623425593853404, + "tokens_seen": 1295974400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067602808425276, + "loss": 2.6319, + "theoretical_loss": 3.5623425593853404, + "tokens_seen": 1295974400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030675025075225674, + "loss": 2.3928, + "theoretical_loss": 3.5623262279186894, + "tokens_seen": 1296039936 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306740220661986, + "loss": 2.5186, + "theoretical_loss": 3.5623098975090564, + "tokens_seen": 1296105472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030673019057171516, + "loss": 2.7353, + "theoretical_loss": 3.5622935681563206, + "tokens_seen": 1296171008 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030672016048144434, + "loss": 2.7635, + "theoretical_loss": 3.56227723986036, + "tokens_seen": 1296236544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067101303911735, + "loss": 2.6512, + "theoretical_loss": 3.562260912621052, + "tokens_seen": 1296302080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067001003009027, + "loss": 2.6434, + "theoretical_loss": 3.5622445864382755, + "tokens_seen": 1296367616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066900702106319, + "loss": 2.5974, + "theoretical_loss": 3.562228261311909, + "tokens_seen": 1296433152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066800401203611, + "loss": 2.635, + "theoretical_loss": 3.56221193724183, + "tokens_seen": 1296498688 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030667001003009025, + "loss": 2.8283, + "theoretical_loss": 3.562195614227917, + "tokens_seen": 1296564224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066599799398195, + "loss": 2.5637, + "theoretical_loss": 3.5621792922700486, + "tokens_seen": 1296629760 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030664994984954866, + "loss": 2.7018, + "theoretical_loss": 3.5621629713681027, + "tokens_seen": 1296695296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030663991975927784, + "loss": 2.8643, + "theoretical_loss": 3.562146651521958, + "tokens_seen": 1296760832 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306629889669007, + "loss": 2.7117, + "theoretical_loss": 3.5621303327314924, + "tokens_seen": 1296826368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066198595787362, + "loss": 2.7569, + "theoretical_loss": 3.5621140149965846, + "tokens_seen": 1296891904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066098294884654, + "loss": 2.7215, + "theoretical_loss": 3.562097698317113, + "tokens_seen": 1296957440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065997993981946, + "loss": 2.75, + "theoretical_loss": 3.5620813826929556, + "tokens_seen": 1297022976 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030658976930792375, + "loss": 2.512, + "theoretical_loss": 3.562065068123992, + "tokens_seen": 1297088512 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306579739217653, + "loss": 2.7242, + "theoretical_loss": 3.562048754610099, + "tokens_seen": 1297154048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065697091273821, + "loss": 2.687, + "theoretical_loss": 3.562032442151156, + "tokens_seen": 1297219584 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030655967903711135, + "loss": 2.6452, + "theoretical_loss": 3.5620161307470415, + "tokens_seen": 1297285120 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030654964894684053, + "loss": 2.5351, + "theoretical_loss": 3.561999820397634, + "tokens_seen": 1297350656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065396188565697, + "loss": 2.7883, + "theoretical_loss": 3.5619835111028113, + "tokens_seen": 1297416192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065295887662989, + "loss": 2.719, + "theoretical_loss": 3.561967202862453, + "tokens_seen": 1297481728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065195586760281, + "loss": 2.6211, + "theoretical_loss": 3.5619508956764374, + "tokens_seen": 1297547264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 716974, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6259679794311523, + "objective/train/theoretical_loss": 3.5619345895446424, + "objective/train/tokens_used": 1318072800, + "theoretical_loss": 3.5619345895446424, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030650952858575725, + "loss": 2.6675, + "theoretical_loss": 3.5619345895446424, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064994984954865, + "loss": 2.767, + "theoretical_loss": 3.5619182844669472, + "tokens_seen": 1297678336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064894684052156, + "loss": 2.5396, + "theoretical_loss": 3.561901980443231, + "tokens_seen": 1297743872 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030647943831494485, + "loss": 2.743, + "theoretical_loss": 3.561885677473371, + "tokens_seen": 1297809408 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030646940822467403, + "loss": 2.6216, + "theoretical_loss": 3.561869375557247, + "tokens_seen": 1297874944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064593781344032, + "loss": 2.7258, + "theoretical_loss": 3.5618530746947368, + "tokens_seen": 1297940480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064493480441324, + "loss": 2.7407, + "theoretical_loss": 3.5618367748857205, + "tokens_seen": 1298006016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064393179538616, + "loss": 2.7233, + "theoretical_loss": 3.561820476130075, + "tokens_seen": 1298071552 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030642928786359076, + "loss": 2.863, + "theoretical_loss": 3.561804178427681, + "tokens_seen": 1298137088 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030641925777332, + "loss": 2.7729, + "theoretical_loss": 3.5617878817784154, + "tokens_seen": 1298202624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064092276830491, + "loss": 2.4632, + "theoretical_loss": 3.5617715861821586, + "tokens_seen": 1298268160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030639919759277835, + "loss": 2.4926, + "theoretical_loss": 3.5617552916387885, + "tokens_seen": 1298333696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063891675025075, + "loss": 2.8324, + "theoretical_loss": 3.561738998148184, + "tokens_seen": 1298399232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063791374122367, + "loss": 2.6675, + "theoretical_loss": 3.561722705710224, + "tokens_seen": 1298464768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063691073219659, + "loss": 2.7435, + "theoretical_loss": 3.561706414324788, + "tokens_seen": 1298530304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063590772316951, + "loss": 2.4899, + "theoretical_loss": 3.5616901239917533, + "tokens_seen": 1298595840 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030634904714142426, + "loss": 2.8086, + "theoretical_loss": 3.5616738347110006, + "tokens_seen": 1298661376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063390170511535, + "loss": 2.5463, + "theoretical_loss": 3.561657546482408, + "tokens_seen": 1298726912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063289869608826, + "loss": 2.5267, + "theoretical_loss": 3.561641259305855, + "tokens_seen": 1298792448 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030631895687061186, + "loss": 2.592, + "theoretical_loss": 3.5616249731812193, + "tokens_seen": 1298857984 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306308926780341, + "loss": 2.571, + "theoretical_loss": 3.5616086881083806, + "tokens_seen": 1298923520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062988966900702, + "loss": 2.6245, + "theoretical_loss": 3.5615924040872184, + "tokens_seen": 1298989056 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030628886659979945, + "loss": 2.5617, + "theoretical_loss": 3.5615761211176116, + "tokens_seen": 1299054592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062788365095286, + "loss": 2.6409, + "theoretical_loss": 3.5615598391994383, + "tokens_seen": 1299120128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062688064192578, + "loss": 2.5904, + "theoretical_loss": 3.5615435583325787, + "tokens_seen": 1299185664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 718007, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9354186058044434, + "objective/train/theoretical_loss": 3.561527278516911, + "objective/train/tokens_used": 1319711200, + "theoretical_loss": 3.561527278516911, + "tokens_seen": 1299251200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030625877632898694, + "loss": 2.6152, + "theoretical_loss": 3.561527278516911, + "tokens_seen": 1299251200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062487462387162, + "loss": 2.5821, + "theoretical_loss": 3.561510999752315, + "tokens_seen": 1299316736 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030623871614844536, + "loss": 2.758, + "theoretical_loss": 3.5614947220386695, + "tokens_seen": 1299382272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030622868605817454, + "loss": 2.6641, + "theoretical_loss": 3.561478445375854, + "tokens_seen": 1299447808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062186559679037, + "loss": 2.8013, + "theoretical_loss": 3.5614621697637476, + "tokens_seen": 1299513344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062086258776329, + "loss": 2.7587, + "theoretical_loss": 3.5614458952022288, + "tokens_seen": 1299578880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061985957873621, + "loss": 2.5621, + "theoretical_loss": 3.561429621691177, + "tokens_seen": 1299644416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061885656970913, + "loss": 2.94, + "theoretical_loss": 3.5614133492304725, + "tokens_seen": 1299709952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030617853560682045, + "loss": 2.7403, + "theoretical_loss": 3.5613970778199935, + "tokens_seen": 1299775488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061685055165497, + "loss": 2.6433, + "theoretical_loss": 3.5613808074596194, + "tokens_seen": 1299841024 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030615847542627886, + "loss": 2.4314, + "theoretical_loss": 3.5613645381492294, + "tokens_seen": 1299906560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030614844533600804, + "loss": 2.788, + "theoretical_loss": 3.5613482698887036, + "tokens_seen": 1299972096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061384152457372, + "loss": 2.676, + "theoretical_loss": 3.5613320026779203, + "tokens_seen": 1300037632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061283851554664, + "loss": 2.8276, + "theoretical_loss": 3.5613157365167596, + "tokens_seen": 1300103168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061183550651956, + "loss": 2.2945, + "theoretical_loss": 3.5612994714051007, + "tokens_seen": 1300168704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061083249749248, + "loss": 2.3596, + "theoretical_loss": 3.5612832073428224, + "tokens_seen": 1300234240 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030609829488465395, + "loss": 2.5757, + "theoretical_loss": 3.5612669443298053, + "tokens_seen": 1300299776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060882647943832, + "loss": 2.6133, + "theoretical_loss": 3.5612506823659276, + "tokens_seen": 1300365312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060782347041123, + "loss": 2.7672, + "theoretical_loss": 3.5612344214510694, + "tokens_seen": 1300430848 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030606820461384155, + "loss": 2.8412, + "theoretical_loss": 3.56121816158511, + "tokens_seen": 1300496384 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030605817452357073, + "loss": 2.555, + "theoretical_loss": 3.561201902767929, + "tokens_seen": 1300561920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060481444332999, + "loss": 2.5812, + "theoretical_loss": 3.561185644999406, + "tokens_seen": 1300627456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060381143430291, + "loss": 2.6127, + "theoretical_loss": 3.5611693882794198, + "tokens_seen": 1300692992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060280842527583, + "loss": 2.6288, + "theoretical_loss": 3.561153132607851, + "tokens_seen": 1300758528 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030601805416248745, + "loss": 2.9774, + "theoretical_loss": 3.5611368779845787, + "tokens_seen": 1300824064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 718956, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0698211193084717, + "objective/train/theoretical_loss": 3.561120624409482, + "objective/train/tokens_used": 1321349600, + "theoretical_loss": 3.561120624409482, + "tokens_seen": 1300889600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060080240722167, + "loss": 2.5853, + "theoretical_loss": 3.561120624409482, + "tokens_seen": 1300889600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059979939819458, + "loss": 2.6357, + "theoretical_loss": 3.5611043718824416, + "tokens_seen": 1300955136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030598796389167505, + "loss": 2.7555, + "theoretical_loss": 3.561088120403336, + "tokens_seen": 1301020672 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030597793380140423, + "loss": 2.7744, + "theoretical_loss": 3.561071869972046, + "tokens_seen": 1301086208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059679037111334, + "loss": 2.5788, + "theoretical_loss": 3.5610556205884505, + "tokens_seen": 1301151744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059578736208626, + "loss": 2.7153, + "theoretical_loss": 3.561039372252429, + "tokens_seen": 1301217280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059478435305918, + "loss": 2.5183, + "theoretical_loss": 3.561023124963861, + "tokens_seen": 1301282816 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030593781344032096, + "loss": 2.8893, + "theoretical_loss": 3.561006878722628, + "tokens_seen": 1301348352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059277833500502, + "loss": 2.8397, + "theoretical_loss": 3.5609906335286077, + "tokens_seen": 1301413888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059177532597793, + "loss": 2.7949, + "theoretical_loss": 3.5609743893816805, + "tokens_seen": 1301479424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030590772316950855, + "loss": 2.8534, + "theoretical_loss": 3.560958146281727, + "tokens_seen": 1301544960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058976930792377, + "loss": 2.5166, + "theoretical_loss": 3.560941904228626, + "tokens_seen": 1301610496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058876629889669, + "loss": 2.7456, + "theoretical_loss": 3.5609256632222577, + "tokens_seen": 1301676032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058776328986961, + "loss": 2.9513, + "theoretical_loss": 3.560909423262502, + "tokens_seen": 1301741568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058676028084253, + "loss": 2.42, + "theoretical_loss": 3.560893184349239, + "tokens_seen": 1301807104 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030585757271815446, + "loss": 2.6008, + "theoretical_loss": 3.560876946482348, + "tokens_seen": 1301872640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058475426278837, + "loss": 2.5304, + "theoretical_loss": 3.560860709661709, + "tokens_seen": 1301938176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058375125376128, + "loss": 2.8498, + "theoretical_loss": 3.5608444738872027, + "tokens_seen": 1302003712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030582748244734206, + "loss": 2.7499, + "theoretical_loss": 3.560828239158708, + "tokens_seen": 1302069248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058174523570712, + "loss": 2.9477, + "theoretical_loss": 3.560812005476105, + "tokens_seen": 1302134784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058074222668004, + "loss": 2.5286, + "theoretical_loss": 3.560795772839275, + "tokens_seen": 1302200320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057973921765296, + "loss": 2.7829, + "theoretical_loss": 3.5607795412480963, + "tokens_seen": 1302265856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057873620862588, + "loss": 2.7022, + "theoretical_loss": 3.56076331070245, + "tokens_seen": 1302331392 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030577733199598796, + "loss": 2.7559, + "theoretical_loss": 3.5607470812022157, + "tokens_seen": 1302396928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030576730190571714, + "loss": 2.6967, + "theoretical_loss": 3.5607308527472736, + "tokens_seen": 1302462464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 719575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2579822540283203, + "objective/train/theoretical_loss": 3.560714625337504, + "objective/train/tokens_used": 1322988000, + "theoretical_loss": 3.560714625337504, + "tokens_seen": 1302528000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057572718154463, + "loss": 2.7297, + "theoretical_loss": 3.560714625337504, + "tokens_seen": 1302528000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030574724172517556, + "loss": 2.74, + "theoretical_loss": 3.5606983989727863, + "tokens_seen": 1302593536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057372116349047, + "loss": 2.5027, + "theoretical_loss": 3.560682173653001, + "tokens_seen": 1302659072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057271815446339, + "loss": 2.5717, + "theoretical_loss": 3.560665949378029, + "tokens_seen": 1302724608 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030571715145436305, + "loss": 2.5871, + "theoretical_loss": 3.560649726147749, + "tokens_seen": 1302790144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057071213640923, + "loss": 2.6637, + "theoretical_loss": 3.560633503962043, + "tokens_seen": 1302855680 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030569709127382147, + "loss": 2.7712, + "theoretical_loss": 3.560617282820789, + "tokens_seen": 1302921216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030568706118355065, + "loss": 2.5889, + "theoretical_loss": 3.560601062723869, + "tokens_seen": 1302986752 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030567703109327983, + "loss": 2.6207, + "theoretical_loss": 3.5605848436711627, + "tokens_seen": 1303052288 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030566700100300906, + "loss": 2.6828, + "theoretical_loss": 3.56056862566255, + "tokens_seen": 1303117824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056569709127382, + "loss": 2.842, + "theoretical_loss": 3.5605524086979123, + "tokens_seen": 1303183360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056469408224674, + "loss": 2.6398, + "theoretical_loss": 3.5605361927771284, + "tokens_seen": 1303248896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030563691073219655, + "loss": 2.6295, + "theoretical_loss": 3.5605199779000793, + "tokens_seen": 1303314432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056268806419258, + "loss": 2.1334, + "theoretical_loss": 3.5605037640666457, + "tokens_seen": 1303379968 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030561685055165497, + "loss": 2.3651, + "theoretical_loss": 3.560487551276707, + "tokens_seen": 1303445504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030560682046138415, + "loss": 2.4331, + "theoretical_loss": 3.560471339530145, + "tokens_seen": 1303511040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030559679037111333, + "loss": 2.7634, + "theoretical_loss": 3.560455128826839, + "tokens_seen": 1303576576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055867602808425, + "loss": 2.8345, + "theoretical_loss": 3.5604389191666694, + "tokens_seen": 1303642112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055767301905717, + "loss": 2.6223, + "theoretical_loss": 3.5604227105495174, + "tokens_seen": 1303707648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030556670010030093, + "loss": 2.6005, + "theoretical_loss": 3.5604065029752627, + "tokens_seen": 1303773184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030555667001003006, + "loss": 2.8058, + "theoretical_loss": 3.560390296443787, + "tokens_seen": 1303838720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055466399197593, + "loss": 2.5682, + "theoretical_loss": 3.5603740909549684, + "tokens_seen": 1303904256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055366098294885, + "loss": 2.6851, + "theoretical_loss": 3.5603578865086902, + "tokens_seen": 1303969792 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030552657973921765, + "loss": 2.783, + "theoretical_loss": 3.560341683104831, + "tokens_seen": 1304035328 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055165496489469, + "loss": 2.6548, + "theoretical_loss": 3.560325480743272, + "tokens_seen": 1304100864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 720457, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.661524534225464, + "objective/train/theoretical_loss": 3.560309279423894, + "objective/train/tokens_used": 1324626400, + "theoretical_loss": 3.560309279423894, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305506519558676, + "loss": 2.7846, + "theoretical_loss": 3.560309279423894, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030549648946840525, + "loss": 2.6931, + "theoretical_loss": 3.560293079146578, + "tokens_seen": 1304231936 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030548645937813443, + "loss": 2.7123, + "theoretical_loss": 3.560276879911203, + "tokens_seen": 1304297472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054764292878636, + "loss": 2.546, + "theoretical_loss": 3.560260681717651, + "tokens_seen": 1304363008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054663991975928, + "loss": 2.5774, + "theoretical_loss": 3.5602444845658026, + "tokens_seen": 1304428544 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305456369107322, + "loss": 2.6763, + "theoretical_loss": 3.560228288455538, + "tokens_seen": 1304494080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030544633901705116, + "loss": 2.6837, + "theoretical_loss": 3.5602120933867383, + "tokens_seen": 1304559616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054363089267804, + "loss": 2.699, + "theoretical_loss": 3.5601958993592833, + "tokens_seen": 1304625152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054262788365095, + "loss": 2.5079, + "theoretical_loss": 3.560179706373055, + "tokens_seen": 1304690688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030541624874623875, + "loss": 2.7306, + "theoretical_loss": 3.5601635144279333, + "tokens_seen": 1304756224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054062186559679, + "loss": 2.712, + "theoretical_loss": 3.5601473235237995, + "tokens_seen": 1304821760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053961885656971, + "loss": 2.6428, + "theoretical_loss": 3.5601311336605344, + "tokens_seen": 1304887296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053861584754263, + "loss": 2.6081, + "theoretical_loss": 3.560114944838018, + "tokens_seen": 1304952832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053761283851555, + "loss": 2.6574, + "theoretical_loss": 3.5600987570561324, + "tokens_seen": 1305018368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030536609829488466, + "loss": 2.8607, + "theoretical_loss": 3.5600825703147576, + "tokens_seen": 1305083904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053560682046139, + "loss": 2.6453, + "theoretical_loss": 3.560066384613774, + "tokens_seen": 1305149440 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305346038114343, + "loss": 2.7926, + "theoretical_loss": 3.560050199953064, + "tokens_seen": 1305214976 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030533600802407226, + "loss": 2.7229, + "theoretical_loss": 3.560034016332507, + "tokens_seen": 1305280512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053259779338014, + "loss": 2.6756, + "theoretical_loss": 3.560017833751985, + "tokens_seen": 1305346048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053159478435306, + "loss": 2.7556, + "theoretical_loss": 3.560001652211378, + "tokens_seen": 1305411584 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053059177532598, + "loss": 2.5928, + "theoretical_loss": 3.559985471710568, + "tokens_seen": 1305477120 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305295887662989, + "loss": 2.6948, + "theoretical_loss": 3.5599692922494355, + "tokens_seen": 1305542656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030528585757271816, + "loss": 2.6325, + "theoretical_loss": 3.559953113827861, + "tokens_seen": 1305608192 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030527582748244734, + "loss": 2.7057, + "theoretical_loss": 3.5599369364457267, + "tokens_seen": 1305673728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052657973921765, + "loss": 2.8171, + "theoretical_loss": 3.559920760102912, + "tokens_seen": 1305739264 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 721070, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7218596935272217, + "objective/train/theoretical_loss": 3.5599045847993, + "objective/train/tokens_used": 1326264800, + "theoretical_loss": 3.5599045847993, + "tokens_seen": 1305804800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030525576730190576, + "loss": 2.6822, + "theoretical_loss": 3.5599045847993, + "tokens_seen": 1305804800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052457372116349, + "loss": 2.6014, + "theoretical_loss": 3.5598884105347706, + "tokens_seen": 1305870336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052357071213641, + "loss": 2.7997, + "theoretical_loss": 3.5598722373092047, + "tokens_seen": 1305935872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030522567703109325, + "loss": 2.6595, + "theoretical_loss": 3.5598560651224838, + "tokens_seen": 1306001408 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052156469408225, + "loss": 2.5854, + "theoretical_loss": 3.5598398939744893, + "tokens_seen": 1306066944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030520561685055167, + "loss": 2.6932, + "theoretical_loss": 3.559823723865102, + "tokens_seen": 1306132480 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030519558676028085, + "loss": 2.6856, + "theoretical_loss": 3.5598075547942027, + "tokens_seen": 1306198016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030518555667001003, + "loss": 2.7419, + "theoretical_loss": 3.5597913867616735, + "tokens_seen": 1306263552 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030517552657973926, + "loss": 2.5488, + "theoretical_loss": 3.559775219767395, + "tokens_seen": 1306329088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051654964894684, + "loss": 2.6137, + "theoretical_loss": 3.559759053811249, + "tokens_seen": 1306394624 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051554663991976, + "loss": 2.8409, + "theoretical_loss": 3.5597428888931164, + "tokens_seen": 1306460160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030514543630892675, + "loss": 2.5936, + "theoretical_loss": 3.559726725012878, + "tokens_seen": 1306525696 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305135406218656, + "loss": 2.612, + "theoretical_loss": 3.559710562170416, + "tokens_seen": 1306591232 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030512537612838517, + "loss": 3.0133, + "theoretical_loss": 3.5596944003656112, + "tokens_seen": 1306656768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030511534603811435, + "loss": 2.7824, + "theoretical_loss": 3.5596782395983455, + "tokens_seen": 1306722304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030510531594784353, + "loss": 2.6491, + "theoretical_loss": 3.5596620798684997, + "tokens_seen": 1306787840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050952858575727, + "loss": 2.6621, + "theoretical_loss": 3.559645921175955, + "tokens_seen": 1306853376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050852557673019, + "loss": 2.5225, + "theoretical_loss": 3.5596297635205927, + "tokens_seen": 1306918912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030507522567703113, + "loss": 2.5159, + "theoretical_loss": 3.5596136069022952, + "tokens_seen": 1306984448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030506519558676026, + "loss": 2.6279, + "theoretical_loss": 3.5595974513209434, + "tokens_seen": 1307049984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050551654964895, + "loss": 2.8104, + "theoretical_loss": 3.559581296776419, + "tokens_seen": 1307115520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050451354062186, + "loss": 2.6946, + "theoretical_loss": 3.5595651432686033, + "tokens_seen": 1307181056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030503510531594785, + "loss": 2.7261, + "theoretical_loss": 3.559548990797377, + "tokens_seen": 1307246592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030502507522567703, + "loss": 2.6394, + "theoretical_loss": 3.559532839362623, + "tokens_seen": 1307312128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050150451354062, + "loss": 2.6643, + "theoretical_loss": 3.5595166889642216, + "tokens_seen": 1307377664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 722360, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.437825918197632, + "objective/train/theoretical_loss": 3.5595005396020554, + "objective/train/tokens_used": 1327903200, + "theoretical_loss": 3.5595005396020554, + "tokens_seen": 1307443200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050050150451354, + "loss": 2.7504, + "theoretical_loss": 3.5595005396020554, + "tokens_seen": 1307443200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030499498495486463, + "loss": 2.8347, + "theoretical_loss": 3.5594843912760052, + "tokens_seen": 1307508736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030498495486459376, + "loss": 2.717, + "theoretical_loss": 3.559468243985953, + "tokens_seen": 1307574272 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304974924774323, + "loss": 2.6803, + "theoretical_loss": 3.5594520977317803, + "tokens_seen": 1307639808 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049648946840521, + "loss": 2.6731, + "theoretical_loss": 3.559435952513369, + "tokens_seen": 1307705344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030495486459378136, + "loss": 2.6494, + "theoretical_loss": 3.5594198083306003, + "tokens_seen": 1307770880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030494483450351054, + "loss": 2.656, + "theoretical_loss": 3.559403665183356, + "tokens_seen": 1307836416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049348044132397, + "loss": 2.803, + "theoretical_loss": 3.559387523071518, + "tokens_seen": 1307901952 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049247743229689, + "loss": 2.7034, + "theoretical_loss": 3.559371381994968, + "tokens_seen": 1307967488 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049147442326981, + "loss": 2.6816, + "theoretical_loss": 3.559355241953588, + "tokens_seen": 1308033024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030490471414242726, + "loss": 2.7026, + "theoretical_loss": 3.5593391029472587, + "tokens_seen": 1308098560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048946840521565, + "loss": 2.7462, + "theoretical_loss": 3.5593229649758626, + "tokens_seen": 1308164096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048846539618856, + "loss": 2.7407, + "theoretical_loss": 3.559306828039282, + "tokens_seen": 1308229632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030487462387161486, + "loss": 2.5191, + "theoretical_loss": 3.559290692137398, + "tokens_seen": 1308295168 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304864593781344, + "loss": 2.8017, + "theoretical_loss": 3.5592745572700917, + "tokens_seen": 1308360704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048545636910732, + "loss": 2.6502, + "theoretical_loss": 3.559258423437247, + "tokens_seen": 1308426240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048445336008024, + "loss": 2.6244, + "theoretical_loss": 3.559242290638744, + "tokens_seen": 1308491776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048345035105316, + "loss": 2.5927, + "theoretical_loss": 3.5592261588744654, + "tokens_seen": 1308557312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030482447342026076, + "loss": 2.4246, + "theoretical_loss": 3.5592100281442924, + "tokens_seen": 1308622848 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030481444332999, + "loss": 2.698, + "theoretical_loss": 3.5591938984481084, + "tokens_seen": 1308688384 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048044132397192, + "loss": 2.8172, + "theoretical_loss": 3.5591777697857934, + "tokens_seen": 1308753920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030479438314944836, + "loss": 2.7269, + "theoretical_loss": 3.559161642157231, + "tokens_seen": 1308819456 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030478435305917754, + "loss": 2.7736, + "theoretical_loss": 3.559145515562302, + "tokens_seen": 1308884992 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047743229689067, + "loss": 2.312, + "theoretical_loss": 3.5591293900008893, + "tokens_seen": 1308950528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030476429287863596, + "loss": 2.5476, + "theoretical_loss": 3.5591132654728743, + "tokens_seen": 1309016064 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 723452, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5018327236175537, + "objective/train/theoretical_loss": 3.55909714197814, + "objective/train/tokens_used": 1329541600, + "theoretical_loss": 3.55909714197814, + "tokens_seen": 1309081600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047542627883651, + "loss": 2.7335, + "theoretical_loss": 3.55909714197814, + "tokens_seen": 1309081600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047442326980943, + "loss": 2.3299, + "theoretical_loss": 3.559081019516567, + "tokens_seen": 1309147136 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030473420260782345, + "loss": 2.7316, + "theoretical_loss": 3.559064898088038, + "tokens_seen": 1309212672 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047241725175527, + "loss": 2.7617, + "theoretical_loss": 3.5590487776924356, + "tokens_seen": 1309278208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030471414242728187, + "loss": 2.7093, + "theoretical_loss": 3.5590326583296417, + "tokens_seen": 1309343744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030470411233701105, + "loss": 2.6649, + "theoretical_loss": 3.559016539999538, + "tokens_seen": 1309409280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030469408224674023, + "loss": 2.6519, + "theoretical_loss": 3.559000422702007, + "tokens_seen": 1309474816 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030468405215646946, + "loss": 2.7074, + "theoretical_loss": 3.558984306436931, + "tokens_seen": 1309540352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046740220661986, + "loss": 2.7133, + "theoretical_loss": 3.558968191204192, + "tokens_seen": 1309605888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046639919759278, + "loss": 2.578, + "theoretical_loss": 3.558952077003672, + "tokens_seen": 1309671424 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030465396188565695, + "loss": 2.6657, + "theoretical_loss": 3.558935963835254, + "tokens_seen": 1309736960 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046439317953862, + "loss": 2.702, + "theoretical_loss": 3.5589198516988194, + "tokens_seen": 1309802496 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030463390170511537, + "loss": 2.6953, + "theoretical_loss": 3.5589037405942507, + "tokens_seen": 1309868032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030462387161484455, + "loss": 2.5696, + "theoretical_loss": 3.55888763052143, + "tokens_seen": 1309933568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030461384152457373, + "loss": 2.7087, + "theoretical_loss": 3.5588715214802407, + "tokens_seen": 1309999104 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046038114343029, + "loss": 2.6059, + "theoretical_loss": 3.558855413470564, + "tokens_seen": 1310064640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045937813440321, + "loss": 2.6829, + "theoretical_loss": 3.558839306492283, + "tokens_seen": 1310130176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030458375125376133, + "loss": 2.5815, + "theoretical_loss": 3.5588232005452793, + "tokens_seen": 1310195712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030457372116349046, + "loss": 2.6662, + "theoretical_loss": 3.5588070956294358, + "tokens_seen": 1310261248 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045636910732197, + "loss": 2.6904, + "theoretical_loss": 3.5587909917446345, + "tokens_seen": 1310326784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045536609829488, + "loss": 2.6112, + "theoretical_loss": 3.558774888890758, + "tokens_seen": 1310392320 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030454363089267805, + "loss": 2.629, + "theoretical_loss": 3.5587587870676893, + "tokens_seen": 1310457856 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030453360080240723, + "loss": 2.5755, + "theoretical_loss": 3.5587426862753104, + "tokens_seen": 1310523392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045235707121364, + "loss": 2.7388, + "theoretical_loss": 3.5587265865135036, + "tokens_seen": 1310588928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045135406218656, + "loss": 2.6498, + "theoretical_loss": 3.558710487782152, + "tokens_seen": 1310654464 + }, + { + "debugging/Self-BLEU-5": 0.39316690427144263, + "debugging/distinct-1-grams": 0.8245694632792082, + "debugging/distinct-2-grams": 0.8227741660491869, + "debugging/entropy-1-grams": 5.538189760337977, + "debugging/entropy-2-grams": 6.130351591764565, + "debugging/length": 468.5, + "debugging/num_segments": 8, + "epoch": 0.4, + "objective/train/docs_used": 724043, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.692194938659668, + "objective/train/theoretical_loss": 3.558694390081137, + "objective/train/tokens_used": 1331180000, + "theoretical_loss": 3.558694390081137, + "tokens_seen": 1310720000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030450351053159483, + "loss": 2.7732, + "theoretical_loss": 3.558694390081137, + "tokens_seen": 1310720000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030449348044132396, + "loss": 2.8132, + "theoretical_loss": 3.5586782934103427, + "tokens_seen": 1310785536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044834503510532, + "loss": 2.7486, + "theoretical_loss": 3.5586621977696504, + "tokens_seen": 1310851072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044734202607823, + "loss": 2.4709, + "theoretical_loss": 3.5586461031589436, + "tokens_seen": 1310916608 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030446339017051156, + "loss": 2.7782, + "theoretical_loss": 3.5586300095781045, + "tokens_seen": 1310982144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030445336008024074, + "loss": 2.6988, + "theoretical_loss": 3.558613917027015, + "tokens_seen": 1311047680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044433299899699, + "loss": 2.5095, + "theoretical_loss": 3.558597825505559, + "tokens_seen": 1311113216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044332998996991, + "loss": 2.4097, + "theoretical_loss": 3.558581735013618, + "tokens_seen": 1311178752 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044232698094283, + "loss": 2.7494, + "theoretical_loss": 3.5585656455510755, + "tokens_seen": 1311244288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030441323971915746, + "loss": 2.6686, + "theoretical_loss": 3.5585495571178143, + "tokens_seen": 1311309824 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044032096288867, + "loss": 2.5388, + "theoretical_loss": 3.5585334697137165, + "tokens_seen": 1311375360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043931795386158, + "loss": 2.7446, + "theoretical_loss": 3.558517383338665, + "tokens_seen": 1311440896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030438314944834506, + "loss": 2.7984, + "theoretical_loss": 3.5585012979925428, + "tokens_seen": 1311506432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043731193580742, + "loss": 2.7733, + "theoretical_loss": 3.5584852136752323, + "tokens_seen": 1311571968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043630892678034, + "loss": 2.7841, + "theoretical_loss": 3.5584691303866167, + "tokens_seen": 1311637504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043530591775326, + "loss": 2.5939, + "theoretical_loss": 3.5584530481265793, + "tokens_seen": 1311703040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043430290872618, + "loss": 2.6253, + "theoretical_loss": 3.5584369668950018, + "tokens_seen": 1311768576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030433299899699097, + "loss": 2.6828, + "theoretical_loss": 3.558420886691767, + "tokens_seen": 1311834112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043229689067202, + "loss": 2.6631, + "theoretical_loss": 3.5584048075167587, + "tokens_seen": 1311899648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030431293881644933, + "loss": 2.6433, + "theoretical_loss": 3.5583887293698595, + "tokens_seen": 1311965184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030430290872617856, + "loss": 2.5721, + "theoretical_loss": 3.5583726522509522, + "tokens_seen": 1312030720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042928786359077, + "loss": 2.8131, + "theoretical_loss": 3.5583565761599196, + "tokens_seen": 1312096256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042828485456369, + "loss": 2.6619, + "theoretical_loss": 3.5583405010966453, + "tokens_seen": 1312161792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042728184553661, + "loss": 2.6893, + "theoretical_loss": 3.5583244270610113, + "tokens_seen": 1312227328 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042627883650953, + "loss": 2.9458, + "theoretical_loss": 3.558308354052901, + "tokens_seen": 1312292864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 725321, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.910378932952881, + "objective/train/theoretical_loss": 3.5582922820721974, + "objective/train/tokens_used": 1332818400, + "theoretical_loss": 3.5582922820721974, + "tokens_seen": 1312358400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030425275827482447, + "loss": 2.7906, + "theoretical_loss": 3.5582922820721974, + "tokens_seen": 1312358400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030424272818455365, + "loss": 2.5895, + "theoretical_loss": 3.558276211118784, + "tokens_seen": 1312423936 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030423269809428283, + "loss": 2.5593, + "theoretical_loss": 3.558260141192543, + "tokens_seen": 1312489472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030422266800401207, + "loss": 2.6732, + "theoretical_loss": 3.558244072293358, + "tokens_seen": 1312555008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042126379137412, + "loss": 2.8627, + "theoretical_loss": 3.558228004421112, + "tokens_seen": 1312620544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030420260782347043, + "loss": 2.6099, + "theoretical_loss": 3.558211937575688, + "tokens_seen": 1312686080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041925777331996, + "loss": 2.781, + "theoretical_loss": 3.5581958717569697, + "tokens_seen": 1312751616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041825476429288, + "loss": 2.6528, + "theoretical_loss": 3.5581798069648394, + "tokens_seen": 1312817152 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030417251755265797, + "loss": 2.6068, + "theoretical_loss": 3.5581637431991804, + "tokens_seen": 1312882688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030416248746238715, + "loss": 2.5357, + "theoretical_loss": 3.5581476804598764, + "tokens_seen": 1312948224 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030415245737211633, + "loss": 2.7134, + "theoretical_loss": 3.55813161874681, + "tokens_seen": 1313013760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030414242728184557, + "loss": 2.8565, + "theoretical_loss": 3.558115558059865, + "tokens_seen": 1313079296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041323971915747, + "loss": 2.6083, + "theoretical_loss": 3.558099498398924, + "tokens_seen": 1313144832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030412236710130393, + "loss": 2.8025, + "theoretical_loss": 3.5580834397638705, + "tokens_seen": 1313210368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030411233701103306, + "loss": 2.8828, + "theoretical_loss": 3.5580673821545883, + "tokens_seen": 1313275904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041023069207623, + "loss": 2.6021, + "theoretical_loss": 3.55805132557096, + "tokens_seen": 1313341440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040922768304915, + "loss": 2.6631, + "theoretical_loss": 3.558035270012869, + "tokens_seen": 1313406976 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030408224674022066, + "loss": 2.5568, + "theoretical_loss": 3.558019215480199, + "tokens_seen": 1313472512 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030407221664994984, + "loss": 2.8165, + "theoretical_loss": 3.5580031619728327, + "tokens_seen": 1313538048 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304062186559679, + "loss": 2.6535, + "theoretical_loss": 3.5579871094906546, + "tokens_seen": 1313603584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030405215646940825, + "loss": 2.8061, + "theoretical_loss": 3.557971058033547, + "tokens_seen": 1313669120 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030404212637913743, + "loss": 2.5655, + "theoretical_loss": 3.557955007601394, + "tokens_seen": 1313734656 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040320962888666, + "loss": 2.7296, + "theoretical_loss": 3.5579389581940783, + "tokens_seen": 1313800192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040220661985958, + "loss": 2.7383, + "theoretical_loss": 3.557922909811484, + "tokens_seen": 1313865728 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030401203610832503, + "loss": 2.7464, + "theoretical_loss": 3.5579068624534944, + "tokens_seen": 1313931264 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 725665, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5255112648010254, + "objective/train/theoretical_loss": 3.5578908161199934, + "objective/train/tokens_used": 1334456800, + "theoretical_loss": 3.5578908161199934, + "tokens_seen": 1313996800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030400200601805416, + "loss": 2.5646, + "theoretical_loss": 3.5578908161199934, + "tokens_seen": 1313996800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039919759277834, + "loss": 2.719, + "theoretical_loss": 3.557874770810863, + "tokens_seen": 1314062336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039819458375125, + "loss": 2.7278, + "theoretical_loss": 3.5578587265259882, + "tokens_seen": 1314127872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030397191574724176, + "loss": 2.6767, + "theoretical_loss": 3.5578426832652523, + "tokens_seen": 1314193408 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030396188565697094, + "loss": 2.9329, + "theoretical_loss": 3.5578266410285386, + "tokens_seen": 1314258944 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039518555667001, + "loss": 2.7439, + "theoretical_loss": 3.557810599815731, + "tokens_seen": 1314324480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039418254764293, + "loss": 2.7347, + "theoretical_loss": 3.5577945596267124, + "tokens_seen": 1314390016 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039317953861585, + "loss": 2.6394, + "theoretical_loss": 3.557778520461367, + "tokens_seen": 1314455552 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030392176529588766, + "loss": 2.4425, + "theoretical_loss": 3.5577624823195784, + "tokens_seen": 1314521088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039117352056169, + "loss": 2.7863, + "theoretical_loss": 3.5577464452012295, + "tokens_seen": 1314586624 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303901705115346, + "loss": 2.6364, + "theoretical_loss": 3.557730409106205, + "tokens_seen": 1314652160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030389167502507526, + "loss": 2.7844, + "theoretical_loss": 3.557714374034389, + "tokens_seen": 1314717696 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038816449348044, + "loss": 2.7259, + "theoretical_loss": 3.557698339985664, + "tokens_seen": 1314783232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038716148445336, + "loss": 2.6794, + "theoretical_loss": 3.5576823069599133, + "tokens_seen": 1314848768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038615847542628, + "loss": 2.8248, + "theoretical_loss": 3.5576662749570227, + "tokens_seen": 1314914304 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303851554663992, + "loss": 2.6524, + "theoretical_loss": 3.557650243976874, + "tokens_seen": 1314979840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030384152457372117, + "loss": 2.862, + "theoretical_loss": 3.5576342140193518, + "tokens_seen": 1315045376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038314944834504, + "loss": 2.8933, + "theoretical_loss": 3.5576181850843405, + "tokens_seen": 1315110912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030382146439317953, + "loss": 2.7301, + "theoretical_loss": 3.557602157171723, + "tokens_seen": 1315176448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030381143430290876, + "loss": 2.6285, + "theoretical_loss": 3.5575861302813827, + "tokens_seen": 1315241984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038014042126379, + "loss": 2.7635, + "theoretical_loss": 3.5575701044132053, + "tokens_seen": 1315307520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037913741223671, + "loss": 2.5867, + "theoretical_loss": 3.557554079567073, + "tokens_seen": 1315373056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037813440320963, + "loss": 2.6651, + "theoretical_loss": 3.5575380557428704, + "tokens_seen": 1315438592 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037713139418255, + "loss": 2.5734, + "theoretical_loss": 3.5575220329404815, + "tokens_seen": 1315504128 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030376128385155467, + "loss": 2.8709, + "theoretical_loss": 3.55750601115979, + "tokens_seen": 1315569664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 726875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6938629150390625, + "objective/train/theoretical_loss": 3.55748999040068, + "objective/train/tokens_used": 1336095200, + "theoretical_loss": 3.55748999040068, + "tokens_seen": 1315635200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030375125376128385, + "loss": 2.8485, + "theoretical_loss": 3.55748999040068, + "tokens_seen": 1315635200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030374122367101303, + "loss": 2.6985, + "theoretical_loss": 3.557473970663035, + "tokens_seen": 1315700736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030373119358074227, + "loss": 2.9056, + "theoretical_loss": 3.5574579519467395, + "tokens_seen": 1315766272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037211634904714, + "loss": 2.5467, + "theoretical_loss": 3.5574419342516777, + "tokens_seen": 1315831808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030371113340020063, + "loss": 2.5505, + "theoretical_loss": 3.557425917577733, + "tokens_seen": 1315897344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037011033099298, + "loss": 2.829, + "theoretical_loss": 3.5574099019247902, + "tokens_seen": 1315962880 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303691073219659, + "loss": 2.763, + "theoretical_loss": 3.557393887292733, + "tokens_seen": 1316028416 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030368104312938817, + "loss": 2.6073, + "theoretical_loss": 3.557377873681445, + "tokens_seen": 1316093952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030367101303911735, + "loss": 2.7531, + "theoretical_loss": 3.5573618610908113, + "tokens_seen": 1316159488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030366098294884653, + "loss": 2.7179, + "theoretical_loss": 3.5573458495207153, + "tokens_seen": 1316225024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030365095285857577, + "loss": 2.7346, + "theoretical_loss": 3.5573298389710413, + "tokens_seen": 1316290560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036409227683049, + "loss": 2.6652, + "theoretical_loss": 3.5573138294416733, + "tokens_seen": 1316356096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030363089267803413, + "loss": 2.7358, + "theoretical_loss": 3.5572978209324964, + "tokens_seen": 1316421632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030362086258776326, + "loss": 2.6463, + "theoretical_loss": 3.5572818134433937, + "tokens_seen": 1316487168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036108324974925, + "loss": 2.7587, + "theoretical_loss": 3.5572658069742498, + "tokens_seen": 1316552704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036008024072217, + "loss": 2.6971, + "theoretical_loss": 3.557249801524949, + "tokens_seen": 1316618240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030359077231695086, + "loss": 2.7197, + "theoretical_loss": 3.5572337970953756, + "tokens_seen": 1316683776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030358074222668004, + "loss": 2.4087, + "theoretical_loss": 3.557217793685414, + "tokens_seen": 1316749312 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035707121364092, + "loss": 2.7399, + "theoretical_loss": 3.557201791294948, + "tokens_seen": 1316814848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035606820461384, + "loss": 2.8805, + "theoretical_loss": 3.5571857899238624, + "tokens_seen": 1316880384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030355065195586763, + "loss": 2.36, + "theoretical_loss": 3.5571697895720416, + "tokens_seen": 1316945920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030354062186559676, + "loss": 2.7899, + "theoretical_loss": 3.5571537902393695, + "tokens_seen": 1317011456 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303530591775326, + "loss": 2.7434, + "theoretical_loss": 3.5571377919257303, + "tokens_seen": 1317076992 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035205616850552, + "loss": 2.9044, + "theoretical_loss": 3.5571217946310094, + "tokens_seen": 1317142528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030351053159478436, + "loss": 2.6923, + "theoretical_loss": 3.5571057983550904, + "tokens_seen": 1317208064 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 727478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8409438133239746, + "objective/train/theoretical_loss": 3.5570898030978584, + "objective/train/tokens_used": 1337733600, + "theoretical_loss": 3.5570898030978584, + "tokens_seen": 1317273600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030350050150451354, + "loss": 2.6263, + "theoretical_loss": 3.5570898030978584, + "tokens_seen": 1317273600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034904714142427, + "loss": 2.693, + "theoretical_loss": 3.5570738088591973, + "tokens_seen": 1317339136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034804413239719, + "loss": 2.6554, + "theoretical_loss": 3.5570578156389914, + "tokens_seen": 1317404672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030347041123370114, + "loss": 2.5147, + "theoretical_loss": 3.5570418234371255, + "tokens_seen": 1317470208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030346038114343026, + "loss": 2.7155, + "theoretical_loss": 3.5570258322534842, + "tokens_seen": 1317535744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034503510531595, + "loss": 2.5846, + "theoretical_loss": 3.5570098420879517, + "tokens_seen": 1317601280 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034403209628886, + "loss": 2.7062, + "theoretical_loss": 3.556993852940413, + "tokens_seen": 1317666816 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030343029087261786, + "loss": 2.7364, + "theoretical_loss": 3.556977864810752, + "tokens_seen": 1317732352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030342026078234704, + "loss": 2.7312, + "theoretical_loss": 3.5569618776988543, + "tokens_seen": 1317797888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034102306920762, + "loss": 2.6489, + "theoretical_loss": 3.5569458916046033, + "tokens_seen": 1317863424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034002006018054, + "loss": 2.5915, + "theoretical_loss": 3.5569299065278845, + "tokens_seen": 1317928960 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033901705115346, + "loss": 2.5329, + "theoretical_loss": 3.556913922468582, + "tokens_seen": 1317994496 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030338014042126377, + "loss": 2.6562, + "theoretical_loss": 3.556897939426581, + "tokens_seen": 1318060032 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303370110330993, + "loss": 2.7461, + "theoretical_loss": 3.5568819574017665, + "tokens_seen": 1318125568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030336008024072213, + "loss": 2.7815, + "theoretical_loss": 3.556865976394022, + "tokens_seen": 1318191104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030335005015045137, + "loss": 2.5943, + "theoretical_loss": 3.5568499964032325, + "tokens_seen": 1318256640 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030334002006018055, + "loss": 2.6025, + "theoretical_loss": 3.5568340174292836, + "tokens_seen": 1318322176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030332998996990973, + "loss": 2.5328, + "theoretical_loss": 3.556818039472059, + "tokens_seen": 1318387712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033199598796389, + "loss": 2.729, + "theoretical_loss": 3.5568020625314443, + "tokens_seen": 1318453248 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033099297893681, + "loss": 2.5643, + "theoretical_loss": 3.5567860866073238, + "tokens_seen": 1318518784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032998996990973, + "loss": 2.741, + "theoretical_loss": 3.5567701116995827, + "tokens_seen": 1318584320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032898696088265, + "loss": 2.7676, + "theoretical_loss": 3.5567541378081056, + "tokens_seen": 1318649856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032798395185557, + "loss": 2.5141, + "theoretical_loss": 3.556738164932777, + "tokens_seen": 1318715392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030326980942828487, + "loss": 2.6577, + "theoretical_loss": 3.5567221930734823, + "tokens_seen": 1318780928 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030325977933801405, + "loss": 2.7262, + "theoretical_loss": 3.5567062222301065, + "tokens_seen": 1318846464 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 728516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7909562587738037, + "objective/train/theoretical_loss": 3.5566902524025332, + "objective/train/tokens_used": 1339372000, + "theoretical_loss": 3.5566902524025332, + "tokens_seen": 1318912000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030324974924774323, + "loss": 2.8554, + "theoretical_loss": 3.5566902524025332, + "tokens_seen": 1318912000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030323971915747247, + "loss": 2.6728, + "theoretical_loss": 3.556674283590649, + "tokens_seen": 1318977536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032296890672016, + "loss": 2.5645, + "theoretical_loss": 3.5566583157943383, + "tokens_seen": 1319043072 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030321965897693083, + "loss": 2.6148, + "theoretical_loss": 3.5566423490134857, + "tokens_seen": 1319108608 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030320962888666, + "loss": 2.7542, + "theoretical_loss": 3.556626383247976, + "tokens_seen": 1319174144 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031995987963892, + "loss": 2.7342, + "theoretical_loss": 3.556610418497695, + "tokens_seen": 1319239680 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030318956870611837, + "loss": 2.7786, + "theoretical_loss": 3.5565944547625277, + "tokens_seen": 1319305216 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030317953861584755, + "loss": 2.486, + "theoretical_loss": 3.556578492042358, + "tokens_seen": 1319370752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030316950852557673, + "loss": 2.591, + "theoretical_loss": 3.5565625303370725, + "tokens_seen": 1319436288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030315947843530597, + "loss": 2.5615, + "theoretical_loss": 3.556546569646555, + "tokens_seen": 1319501824 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031494483450351, + "loss": 2.4469, + "theoretical_loss": 3.5565306099706904, + "tokens_seen": 1319567360 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030313941825476433, + "loss": 2.6545, + "theoretical_loss": 3.5565146513093655, + "tokens_seen": 1319632896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030312938816449346, + "loss": 2.6957, + "theoretical_loss": 3.556498693662464, + "tokens_seen": 1319698432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031193580742227, + "loss": 2.5422, + "theoretical_loss": 3.556482737029871, + "tokens_seen": 1319763968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031093279839519, + "loss": 2.7761, + "theoretical_loss": 3.5564667814114728, + "tokens_seen": 1319829504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030309929789368106, + "loss": 2.7011, + "theoretical_loss": 3.5564508268071533, + "tokens_seen": 1319895040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030308926780341024, + "loss": 2.5986, + "theoretical_loss": 3.5564348732167987, + "tokens_seen": 1319960576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030792377131394, + "loss": 2.7712, + "theoretical_loss": 3.5564189206402936, + "tokens_seen": 1320026112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030692076228686, + "loss": 2.5667, + "theoretical_loss": 3.5564029690775234, + "tokens_seen": 1320091648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030305917753259783, + "loss": 2.5837, + "theoretical_loss": 3.5563870185283735, + "tokens_seen": 1320157184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030304914744232696, + "loss": 2.8657, + "theoretical_loss": 3.556371068992729, + "tokens_seen": 1320222720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030391173520562, + "loss": 2.734, + "theoretical_loss": 3.5563551204704753, + "tokens_seen": 1320288256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030290872617854, + "loss": 2.4422, + "theoretical_loss": 3.5563391729614975, + "tokens_seen": 1320353792 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030301905717151456, + "loss": 2.8178, + "theoretical_loss": 3.556323226465681, + "tokens_seen": 1320419328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030300902708124374, + "loss": 2.7261, + "theoretical_loss": 3.556307280982912, + "tokens_seen": 1320484864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 729780, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1676318645477295, + "objective/train/theoretical_loss": 3.556291336513074, + "objective/train/tokens_used": 1341010400, + "theoretical_loss": 3.556291336513074, + "tokens_seen": 1320550400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029989969909729, + "loss": 2.4284, + "theoretical_loss": 3.556291336513074, + "tokens_seen": 1320550400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029889669007021, + "loss": 2.6546, + "theoretical_loss": 3.5562753930560547, + "tokens_seen": 1320615936 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030297893681043134, + "loss": 2.6635, + "theoretical_loss": 3.556259450611738, + "tokens_seen": 1320681472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030296890672016046, + "loss": 2.6725, + "theoretical_loss": 3.5562435091800095, + "tokens_seen": 1320747008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029588766298897, + "loss": 2.7965, + "theoretical_loss": 3.5562275687607547, + "tokens_seen": 1320812544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029488465396188, + "loss": 2.6281, + "theoretical_loss": 3.5562116293538595, + "tokens_seen": 1320878080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030293881644934806, + "loss": 2.5591, + "theoretical_loss": 3.5561956909592087, + "tokens_seen": 1320943616 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030292878635907724, + "loss": 2.8417, + "theoretical_loss": 3.5561797535766884, + "tokens_seen": 1321009152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029187562688064, + "loss": 2.5182, + "theoretical_loss": 3.5561638172061834, + "tokens_seen": 1321074688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029087261785356, + "loss": 2.7444, + "theoretical_loss": 3.5561478818475805, + "tokens_seen": 1321140224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028986960882648, + "loss": 2.7431, + "theoretical_loss": 3.556131947500764, + "tokens_seen": 1321205760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030288866599799397, + "loss": 2.7926, + "theoretical_loss": 3.55611601416562, + "tokens_seen": 1321271296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028786359077232, + "loss": 2.8462, + "theoretical_loss": 3.5561000818420343, + "tokens_seen": 1321336832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030286860581745233, + "loss": 2.6434, + "theoretical_loss": 3.556084150529892, + "tokens_seen": 1321402368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030285857572718157, + "loss": 2.2269, + "theoretical_loss": 3.5560682202290783, + "tokens_seen": 1321467904 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030284854563691075, + "loss": 2.6968, + "theoretical_loss": 3.5560522909394807, + "tokens_seen": 1321533440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030283851554663993, + "loss": 2.8215, + "theoretical_loss": 3.5560363626609828, + "tokens_seen": 1321598976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028284854563691, + "loss": 2.6064, + "theoretical_loss": 3.5560204353934717, + "tokens_seen": 1321664512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028184553660983, + "loss": 2.6011, + "theoretical_loss": 3.5560045091368324, + "tokens_seen": 1321730048 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030280842527582747, + "loss": 2.6131, + "theoretical_loss": 3.555988583890951, + "tokens_seen": 1321795584 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027983951855567, + "loss": 2.694, + "theoretical_loss": 3.555972659655713, + "tokens_seen": 1321861120 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030278836509528583, + "loss": 2.5547, + "theoretical_loss": 3.5559567364310043, + "tokens_seen": 1321926656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030277833500501507, + "loss": 2.676, + "theoretical_loss": 3.5559408142167106, + "tokens_seen": 1321992192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027683049147442, + "loss": 2.7288, + "theoretical_loss": 3.555924893012717, + "tokens_seen": 1322057728 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030275827482447343, + "loss": 2.7041, + "theoretical_loss": 3.5559089728189113, + "tokens_seen": 1322123264 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 730226, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.751304864883423, + "objective/train/theoretical_loss": 3.555893053635177, + "objective/train/tokens_used": 1342648800, + "theoretical_loss": 3.555893053635177, + "tokens_seen": 1322188800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027482447342026, + "loss": 2.8815, + "theoretical_loss": 3.555893053635177, + "tokens_seen": 1322188800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027382146439318, + "loss": 2.6246, + "theoretical_loss": 3.555877135461401, + "tokens_seen": 1322254336 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302728184553661, + "loss": 2.6019, + "theoretical_loss": 3.55586121829747, + "tokens_seen": 1322319872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027181544633902, + "loss": 2.5323, + "theoretical_loss": 3.5558453021432683, + "tokens_seen": 1322385408 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030270812437311934, + "loss": 2.497, + "theoretical_loss": 3.555829386998683, + "tokens_seen": 1322450944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030269809428284857, + "loss": 2.8508, + "theoretical_loss": 3.5558134728635995, + "tokens_seen": 1322516480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026880641925777, + "loss": 2.629, + "theoretical_loss": 3.5557975597379037, + "tokens_seen": 1322582016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030267803410230693, + "loss": 2.5966, + "theoretical_loss": 3.555781647621482, + "tokens_seen": 1322647552 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026680040120361, + "loss": 2.803, + "theoretical_loss": 3.55576573651422, + "tokens_seen": 1322713088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026579739217653, + "loss": 2.7159, + "theoretical_loss": 3.5557498264160037, + "tokens_seen": 1322778624 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026479438314945, + "loss": 2.7146, + "theoretical_loss": 3.5557339173267195, + "tokens_seen": 1322844160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030263791374122366, + "loss": 2.6778, + "theoretical_loss": 3.555718009246253, + "tokens_seen": 1322909696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030262788365095284, + "loss": 2.6243, + "theoretical_loss": 3.5557021021744903, + "tokens_seen": 1322975232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026178535606821, + "loss": 2.7398, + "theoretical_loss": 3.5556861961113175, + "tokens_seen": 1323040768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026078234704112, + "loss": 2.6839, + "theoretical_loss": 3.555670291056621, + "tokens_seen": 1323106304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030259779338014044, + "loss": 2.6276, + "theoretical_loss": 3.5556543870102866, + "tokens_seen": 1323171840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030258776328986956, + "loss": 2.8223, + "theoretical_loss": 3.5556384839722006, + "tokens_seen": 1323237376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025777331995988, + "loss": 2.6482, + "theoretical_loss": 3.5556225819422496, + "tokens_seen": 1323302912 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302567703109328, + "loss": 2.8187, + "theoretical_loss": 3.5556066809203184, + "tokens_seen": 1323368448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030255767301905716, + "loss": 2.6949, + "theoretical_loss": 3.5555907809062948, + "tokens_seen": 1323433984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025476429287864, + "loss": 2.4339, + "theoretical_loss": 3.5555748819000637, + "tokens_seen": 1323499520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025376128385156, + "loss": 2.6973, + "theoretical_loss": 3.555558983901512, + "tokens_seen": 1323565056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030252758274824476, + "loss": 2.6124, + "theoretical_loss": 3.555543086910526, + "tokens_seen": 1323630592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030251755265797394, + "loss": 2.7644, + "theoretical_loss": 3.5555271909269917, + "tokens_seen": 1323696128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025075225677031, + "loss": 2.8019, + "theoretical_loss": 3.5555112959507955, + "tokens_seen": 1323761664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 731155, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7156481742858887, + "objective/train/theoretical_loss": 3.5554954019818235, + "objective/train/tokens_used": 1344287200, + "theoretical_loss": 3.5554954019818235, + "tokens_seen": 1323827200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024974924774323, + "loss": 2.7315, + "theoretical_loss": 3.5554954019818235, + "tokens_seen": 1323827200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030248746238716154, + "loss": 2.8048, + "theoretical_loss": 3.5554795090199622, + "tokens_seen": 1323892736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030247743229689066, + "loss": 2.7554, + "theoretical_loss": 3.555463617065098, + "tokens_seen": 1323958272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024674022066199, + "loss": 2.6749, + "theoretical_loss": 3.5554477261171176, + "tokens_seen": 1324023808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030245737211634903, + "loss": 2.4962, + "theoretical_loss": 3.5554318361759063, + "tokens_seen": 1324089344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030244734202607826, + "loss": 2.7971, + "theoretical_loss": 3.555415947241351, + "tokens_seen": 1324154880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030243731193580744, + "loss": 2.6869, + "theoretical_loss": 3.555400059313339, + "tokens_seen": 1324220416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024272818455366, + "loss": 2.8608, + "theoretical_loss": 3.555384172391755, + "tokens_seen": 1324285952 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024172517552658, + "loss": 2.7737, + "theoretical_loss": 3.5553682864764875, + "tokens_seen": 1324351488 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302407221664995, + "loss": 2.5956, + "theoretical_loss": 3.555352401567421, + "tokens_seen": 1324417024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030239719157472417, + "loss": 2.5396, + "theoretical_loss": 3.5553365176644434, + "tokens_seen": 1324482560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023871614844534, + "loss": 2.5868, + "theoretical_loss": 3.5553206347674404, + "tokens_seen": 1324548096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030237713139418253, + "loss": 2.6205, + "theoretical_loss": 3.555304752876299, + "tokens_seen": 1324613632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030236710130391177, + "loss": 2.673, + "theoretical_loss": 3.5552888719909053, + "tokens_seen": 1324679168 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030235707121364095, + "loss": 2.7001, + "theoretical_loss": 3.5552729921111466, + "tokens_seen": 1324744704 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030234704112337013, + "loss": 2.7891, + "theoretical_loss": 3.555257113236908, + "tokens_seen": 1324810240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023370110330993, + "loss": 2.747, + "theoretical_loss": 3.5552412353680776, + "tokens_seen": 1324875776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023269809428285, + "loss": 2.6373, + "theoretical_loss": 3.5552253585045417, + "tokens_seen": 1324941312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030231695085255767, + "loss": 2.5577, + "theoretical_loss": 3.5552094826461866, + "tokens_seen": 1325006848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023069207622869, + "loss": 2.6893, + "theoretical_loss": 3.5551936077928987, + "tokens_seen": 1325072384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030229689067201603, + "loss": 2.5837, + "theoretical_loss": 3.555177733944565, + "tokens_seen": 1325137920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030228686058174527, + "loss": 2.7776, + "theoretical_loss": 3.5551618611010722, + "tokens_seen": 1325203456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022768304914744, + "loss": 2.6919, + "theoretical_loss": 3.555145989262307, + "tokens_seen": 1325268992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030226680040120363, + "loss": 2.7789, + "theoretical_loss": 3.5551301184281567, + "tokens_seen": 1325334528 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022567703109328, + "loss": 2.6184, + "theoretical_loss": 3.5551142485985068, + "tokens_seen": 1325400064 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 731791, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.585141658782959, + "objective/train/theoretical_loss": 3.555098379773245, + "objective/train/tokens_used": 1345925600, + "theoretical_loss": 3.555098379773245, + "tokens_seen": 1325465600 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302246740220662, + "loss": 2.6396, + "theoretical_loss": 3.555098379773245, + "tokens_seen": 1325465600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022367101303912, + "loss": 2.6297, + "theoretical_loss": 3.555082511952258, + "tokens_seen": 1325531136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022266800401204, + "loss": 2.7135, + "theoretical_loss": 3.555066645135432, + "tokens_seen": 1325596672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030221664994984954, + "loss": 2.6646, + "theoretical_loss": 3.555050779322654, + "tokens_seen": 1325662208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030220661985957877, + "loss": 2.6371, + "theoretical_loss": 3.5550349145138114, + "tokens_seen": 1325727744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021965897693079, + "loss": 2.6964, + "theoretical_loss": 3.5550190507087907, + "tokens_seen": 1325793280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030218655967903713, + "loss": 2.8007, + "theoretical_loss": 3.5550031879074786, + "tokens_seen": 1325858816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021765295887663, + "loss": 2.6598, + "theoretical_loss": 3.554987326109762, + "tokens_seen": 1325924352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021664994984955, + "loss": 2.7457, + "theoretical_loss": 3.5549714653155284, + "tokens_seen": 1325989888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021564694082247, + "loss": 2.7275, + "theoretical_loss": 3.554955605524664, + "tokens_seen": 1326055424 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030214643931795386, + "loss": 2.7028, + "theoretical_loss": 3.5549397467370563, + "tokens_seen": 1326120960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030213640922768304, + "loss": 2.7375, + "theoretical_loss": 3.554923888952591, + "tokens_seen": 1326186496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021263791374123, + "loss": 2.6846, + "theoretical_loss": 3.5549080321711575, + "tokens_seen": 1326252032 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021163490471414, + "loss": 2.7879, + "theoretical_loss": 3.5548921763926407, + "tokens_seen": 1326317568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030210631895687064, + "loss": 2.6558, + "theoretical_loss": 3.554876321616928, + "tokens_seen": 1326383104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030209628886659976, + "loss": 2.7538, + "theoretical_loss": 3.5548604678439064, + "tokens_seen": 1326448640 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302086258776329, + "loss": 2.5872, + "theoretical_loss": 3.5548446150734643, + "tokens_seen": 1326514176 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020762286860582, + "loss": 2.7702, + "theoretical_loss": 3.554828763305487, + "tokens_seen": 1326579712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030206619859578736, + "loss": 2.672, + "theoretical_loss": 3.5548129125398624, + "tokens_seen": 1326645248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030205616850551654, + "loss": 2.7365, + "theoretical_loss": 3.5547970627764776, + "tokens_seen": 1326710784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020461384152458, + "loss": 2.6226, + "theoretical_loss": 3.5547812140152195, + "tokens_seen": 1326776320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020361083249749, + "loss": 2.6412, + "theoretical_loss": 3.5547653662559755, + "tokens_seen": 1326841856 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030202607823470414, + "loss": 2.6594, + "theoretical_loss": 3.5547495194986327, + "tokens_seen": 1326907392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030201604814443327, + "loss": 2.6627, + "theoretical_loss": 3.554733673743078, + "tokens_seen": 1326972928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020060180541625, + "loss": 2.5631, + "theoretical_loss": 3.554717828989199, + "tokens_seen": 1327038464 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 732925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.762134075164795, + "objective/train/theoretical_loss": 3.554701985236883, + "objective/train/tokens_used": 1347564000, + "theoretical_loss": 3.554701985236883, + "tokens_seen": 1327104000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003019959879638917, + "loss": 2.5826, + "theoretical_loss": 3.554701985236883, + "tokens_seen": 1327104000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030198595787362087, + "loss": 2.6966, + "theoretical_loss": 3.5546861424860166, + "tokens_seen": 1327169536 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030197592778335005, + "loss": 2.7382, + "theoretical_loss": 3.554670300736487, + "tokens_seen": 1327235072 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030196589769307923, + "loss": 2.68, + "theoretical_loss": 3.554654459988183, + "tokens_seen": 1327300608 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003019558676028084, + "loss": 2.7091, + "theoretical_loss": 3.55463862024099, + "tokens_seen": 1327366144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030194583751253764, + "loss": 2.8205, + "theoretical_loss": 3.554622781494796, + "tokens_seen": 1327431680 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030193580742226677, + "loss": 3.0017, + "theoretical_loss": 3.554606943749489, + "tokens_seen": 1327497216 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301925777331996, + "loss": 2.7315, + "theoretical_loss": 3.5545911070049554, + "tokens_seen": 1327562752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030191574724172513, + "loss": 2.74, + "theoretical_loss": 3.5545752712610836, + "tokens_seen": 1327628288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030190571715145437, + "loss": 2.6388, + "theoretical_loss": 3.5545594365177595, + "tokens_seen": 1327693824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030189568706118355, + "loss": 2.6028, + "theoretical_loss": 3.5545436027748716, + "tokens_seen": 1327759360 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030188565697091273, + "loss": 2.6851, + "theoretical_loss": 3.554527770032307, + "tokens_seen": 1327824896 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018756268806419, + "loss": 2.7161, + "theoretical_loss": 3.554511938289953, + "tokens_seen": 1327890432 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030186559679037115, + "loss": 2.6321, + "theoretical_loss": 3.5544961075476977, + "tokens_seen": 1327955968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018555667001003, + "loss": 2.6999, + "theoretical_loss": 3.5544802778054274, + "tokens_seen": 1328021504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018455366098295, + "loss": 2.5249, + "theoretical_loss": 3.554464449063031, + "tokens_seen": 1328087040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030183550651955864, + "loss": 2.5905, + "theoretical_loss": 3.554448621320395, + "tokens_seen": 1328152576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030182547642928787, + "loss": 2.7502, + "theoretical_loss": 3.5544327945774072, + "tokens_seen": 1328218112 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030181544633901705, + "loss": 2.7522, + "theoretical_loss": 3.554416968833955, + "tokens_seen": 1328283648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030180541624874623, + "loss": 2.4335, + "theoretical_loss": 3.554401144089926, + "tokens_seen": 1328349184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030179538615847547, + "loss": 2.753, + "theoretical_loss": 3.554385320345209, + "tokens_seen": 1328414720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017853560682046, + "loss": 2.7503, + "theoretical_loss": 3.5543694975996893, + "tokens_seen": 1328480256 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030177532597793383, + "loss": 2.6822, + "theoretical_loss": 3.554353675853256, + "tokens_seen": 1328545792 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301765295887663, + "loss": 2.7244, + "theoretical_loss": 3.554337855105797, + "tokens_seen": 1328611328 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017552657973922, + "loss": 2.6174, + "theoretical_loss": 3.554322035357199, + "tokens_seen": 1328676864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 733999, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7849137783050537, + "objective/train/theoretical_loss": 3.5543062166073502, + "objective/train/tokens_used": 1349202400, + "theoretical_loss": 3.5543062166073502, + "tokens_seen": 1328742400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017452357071214, + "loss": 2.7786, + "theoretical_loss": 3.5543062166073502, + "tokens_seen": 1328742400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017352056168506, + "loss": 2.9597, + "theoretical_loss": 3.554290398856138, + "tokens_seen": 1328807936 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030172517552657974, + "loss": 2.5949, + "theoretical_loss": 3.5542745821034503, + "tokens_seen": 1328873472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030171514543630897, + "loss": 2.6613, + "theoretical_loss": 3.554258766349175, + "tokens_seen": 1328939008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017051153460381, + "loss": 2.5457, + "theoretical_loss": 3.5542429515931993, + "tokens_seen": 1329004544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030169508525576733, + "loss": 2.7014, + "theoretical_loss": 3.5542271378354116, + "tokens_seen": 1329070080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016850551654965, + "loss": 2.8066, + "theoretical_loss": 3.5542113250756993, + "tokens_seen": 1329135616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016750250752257, + "loss": 2.824, + "theoretical_loss": 3.5541955133139504, + "tokens_seen": 1329201152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016649949849549, + "loss": 2.6049, + "theoretical_loss": 3.5541797025500523, + "tokens_seen": 1329266688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030165496489468406, + "loss": 2.7338, + "theoretical_loss": 3.554163892783894, + "tokens_seen": 1329332224 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030164493480441324, + "loss": 2.7905, + "theoretical_loss": 3.5541480840153614, + "tokens_seen": 1329397760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016349047141425, + "loss": 2.6749, + "theoretical_loss": 3.5541322762443444, + "tokens_seen": 1329463296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016248746238716, + "loss": 2.5686, + "theoretical_loss": 3.5541164694707295, + "tokens_seen": 1329528832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030161484453360084, + "loss": 2.7111, + "theoretical_loss": 3.5541006636944052, + "tokens_seen": 1329594368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030160481444332996, + "loss": 2.6742, + "theoretical_loss": 3.5540848589152594, + "tokens_seen": 1329659904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015947843530592, + "loss": 2.6322, + "theoretical_loss": 3.55406905513318, + "tokens_seen": 1329725440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015847542627884, + "loss": 2.7064, + "theoretical_loss": 3.5540532523480546, + "tokens_seen": 1329790976 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030157472417251756, + "loss": 2.7721, + "theoretical_loss": 3.554037450559772, + "tokens_seen": 1329856512 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030156469408224674, + "loss": 2.6562, + "theoretical_loss": 3.554021649768219, + "tokens_seen": 1329922048 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301554663991976, + "loss": 2.7299, + "theoretical_loss": 3.5540058499732847, + "tokens_seen": 1329987584 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015446339017051, + "loss": 2.7501, + "theoretical_loss": 3.5539900511748566, + "tokens_seen": 1330053120 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030153460381143434, + "loss": 2.6478, + "theoretical_loss": 3.553974253372823, + "tokens_seen": 1330118656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030152457372116347, + "loss": 2.8084, + "theoretical_loss": 3.553958456567072, + "tokens_seen": 1330184192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015145436308927, + "loss": 2.5444, + "theoretical_loss": 3.5539426607574915, + "tokens_seen": 1330249728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015045135406219, + "loss": 2.2331, + "theoretical_loss": 3.5539268659439696, + "tokens_seen": 1330315264 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 734673, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9271119832992554, + "objective/train/theoretical_loss": 3.553911072126394, + "objective/train/tokens_used": 1350840800, + "theoretical_loss": 3.553911072126394, + "tokens_seen": 1330380800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030149448345035107, + "loss": 2.3999, + "theoretical_loss": 3.553911072126394, + "tokens_seen": 1330380800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030148445336008025, + "loss": 2.5087, + "theoretical_loss": 3.553895279304654, + "tokens_seen": 1330446336 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030147442326980943, + "loss": 2.7373, + "theoretical_loss": 3.553879487478637, + "tokens_seen": 1330511872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014643931795386, + "loss": 2.8742, + "theoretical_loss": 3.5538636966482313, + "tokens_seen": 1330577408 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030145436308926784, + "loss": 2.7182, + "theoretical_loss": 3.553847906813325, + "tokens_seen": 1330642944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030144433299899697, + "loss": 2.5288, + "theoretical_loss": 3.553832117973806, + "tokens_seen": 1330708480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014343029087262, + "loss": 2.5994, + "theoretical_loss": 3.553816330129563, + "tokens_seen": 1330774016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030142427281845533, + "loss": 2.6242, + "theoretical_loss": 3.553800543280485, + "tokens_seen": 1330839552 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030141424272818457, + "loss": 2.4113, + "theoretical_loss": 3.5537847574264587, + "tokens_seen": 1330905088 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030140421263791375, + "loss": 2.6611, + "theoretical_loss": 3.553768972567373, + "tokens_seen": 1330970624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030139418254764293, + "loss": 2.867, + "theoretical_loss": 3.553753188703117, + "tokens_seen": 1331036160 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013841524573721, + "loss": 2.8117, + "theoretical_loss": 3.5537374058335782, + "tokens_seen": 1331101696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030137412236710135, + "loss": 2.6837, + "theoretical_loss": 3.553721623958645, + "tokens_seen": 1331167232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013640922768305, + "loss": 2.5018, + "theoretical_loss": 3.5537058430782054, + "tokens_seen": 1331232768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013540621865597, + "loss": 2.5259, + "theoretical_loss": 3.553690063192149, + "tokens_seen": 1331298304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030134403209628884, + "loss": 2.9599, + "theoretical_loss": 3.553674284300363, + "tokens_seen": 1331363840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030133400200601807, + "loss": 2.8089, + "theoretical_loss": 3.553658506402736, + "tokens_seen": 1331429376 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030132397191574725, + "loss": 2.7761, + "theoretical_loss": 3.553642729499157, + "tokens_seen": 1331494912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030131394182547643, + "loss": 2.6749, + "theoretical_loss": 3.5536269535895144, + "tokens_seen": 1331560448 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013039117352056, + "loss": 2.6557, + "theoretical_loss": 3.5536111786736964, + "tokens_seen": 1331625984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012938816449348, + "loss": 2.7507, + "theoretical_loss": 3.553595404751591, + "tokens_seen": 1331691520 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301283851554664, + "loss": 2.7159, + "theoretical_loss": 3.5535796318230872, + "tokens_seen": 1331757056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012738214643932, + "loss": 2.6725, + "theoretical_loss": 3.553563859888074, + "tokens_seen": 1331822592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030126379137412234, + "loss": 2.4583, + "theoretical_loss": 3.5535480889464393, + "tokens_seen": 1331888128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012537612838516, + "loss": 2.5396, + "theoretical_loss": 3.5535323189980716, + "tokens_seen": 1331953664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 735529, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.719951629638672, + "objective/train/theoretical_loss": 3.55351655004286, + "objective/train/tokens_used": 1352479200, + "theoretical_loss": 3.55351655004286, + "tokens_seen": 1332019200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012437311935807, + "loss": 2.7935, + "theoretical_loss": 3.55351655004286, + "tokens_seen": 1332019200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030123370110330994, + "loss": 2.939, + "theoretical_loss": 3.5535007820806923, + "tokens_seen": 1332084736 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012236710130391, + "loss": 2.7108, + "theoretical_loss": 3.553485015111458, + "tokens_seen": 1332150272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012136409227683, + "loss": 2.6529, + "theoretical_loss": 3.5534692491350452, + "tokens_seen": 1332215808 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012036108324975, + "loss": 2.8294, + "theoretical_loss": 3.5534534841513428, + "tokens_seen": 1332281344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011935807422267, + "loss": 2.7404, + "theoretical_loss": 3.553437720160239, + "tokens_seen": 1332346880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030118355065195584, + "loss": 2.7931, + "theoretical_loss": 3.553421957161623, + "tokens_seen": 1332412416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011735205616851, + "loss": 2.7708, + "theoretical_loss": 3.5534061951553837, + "tokens_seen": 1332477952 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011634904714142, + "loss": 2.6398, + "theoretical_loss": 3.5533904341414093, + "tokens_seen": 1332543488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030115346038114344, + "loss": 2.6252, + "theoretical_loss": 3.5533746741195884, + "tokens_seen": 1332609024 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011434302908726, + "loss": 2.6751, + "theoretical_loss": 3.5533589150898104, + "tokens_seen": 1332674560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011334002006018, + "loss": 2.6118, + "theoretical_loss": 3.5533431570519634, + "tokens_seen": 1332740096 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301123370110331, + "loss": 2.7787, + "theoretical_loss": 3.5533274000059363, + "tokens_seen": 1332805632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030111334002006016, + "loss": 2.7079, + "theoretical_loss": 3.5533116439516186, + "tokens_seen": 1332871168 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030110330992978935, + "loss": 2.7413, + "theoretical_loss": 3.5532958888888984, + "tokens_seen": 1332936704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010932798395186, + "loss": 2.6334, + "theoretical_loss": 3.5532801348176646, + "tokens_seen": 1333002240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010832497492477, + "loss": 2.684, + "theoretical_loss": 3.5532643817378067, + "tokens_seen": 1333067776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030107321965897694, + "loss": 2.5295, + "theoretical_loss": 3.5532486296492127, + "tokens_seen": 1333133312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030106318956870607, + "loss": 2.7846, + "theoretical_loss": 3.553232878551772, + "tokens_seen": 1333198848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010531594784353, + "loss": 2.6775, + "theoretical_loss": 3.553217128445374, + "tokens_seen": 1333264384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030104312938816454, + "loss": 2.6434, + "theoretical_loss": 3.5532013793299067, + "tokens_seen": 1333329920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030103309929789367, + "loss": 2.5548, + "theoretical_loss": 3.5531856312052597, + "tokens_seen": 1333395456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010230692076229, + "loss": 2.7114, + "theoretical_loss": 3.553169884071321, + "tokens_seen": 1333460992 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010130391173521, + "loss": 2.6786, + "theoretical_loss": 3.553154137927981, + "tokens_seen": 1333526528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030100300902708127, + "loss": 2.4968, + "theoretical_loss": 3.5531383927751277, + "tokens_seen": 1333592064 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 735803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6573450565338135, + "objective/train/theoretical_loss": 3.5531226486126504, + "objective/train/tokens_used": 1354117600, + "theoretical_loss": 3.5531226486126504, + "tokens_seen": 1333657600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030099297893681045, + "loss": 2.5315, + "theoretical_loss": 3.5531226486126504, + "tokens_seen": 1333657600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030098294884653963, + "loss": 2.6829, + "theoretical_loss": 3.5531069054404383, + "tokens_seen": 1333723136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003009729187562688, + "loss": 2.8954, + "theoretical_loss": 3.5530911632583804, + "tokens_seen": 1333788672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030096288866599804, + "loss": 2.9845, + "theoretical_loss": 3.5530754220663656, + "tokens_seen": 1333854208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030095285857572717, + "loss": 2.7962, + "theoretical_loss": 3.553059681864283, + "tokens_seen": 1333919744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003009428284854564, + "loss": 2.6451, + "theoretical_loss": 3.553043942652022, + "tokens_seen": 1333985280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030093279839518553, + "loss": 2.913, + "theoretical_loss": 3.553028204429471, + "tokens_seen": 1334050816 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030092276830491477, + "loss": 2.4407, + "theoretical_loss": 3.5530124671965204, + "tokens_seen": 1334116352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030091273821464395, + "loss": 2.6616, + "theoretical_loss": 3.5529967309530583, + "tokens_seen": 1334181888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030090270812437313, + "loss": 2.4699, + "theoretical_loss": 3.5529809956989746, + "tokens_seen": 1334247424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008926780341023, + "loss": 2.6506, + "theoretical_loss": 3.5529652614341583, + "tokens_seen": 1334312960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030088264794383155, + "loss": 2.5312, + "theoretical_loss": 3.5529495281584977, + "tokens_seen": 1334378496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008726178535607, + "loss": 2.53, + "theoretical_loss": 3.5529337958718834, + "tokens_seen": 1334444032 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008625877632899, + "loss": 2.5769, + "theoretical_loss": 3.552918064574204, + "tokens_seen": 1334509568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030085255767301904, + "loss": 2.8111, + "theoretical_loss": 3.5529023342653483, + "tokens_seen": 1334575104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030084252758274827, + "loss": 2.7617, + "theoretical_loss": 3.552886604945207, + "tokens_seen": 1334640640 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030083249749247745, + "loss": 2.5592, + "theoretical_loss": 3.5528708766136683, + "tokens_seen": 1334706176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030082246740220663, + "loss": 2.5869, + "theoretical_loss": 3.5528551492706217, + "tokens_seen": 1334771712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008124373119358, + "loss": 2.8061, + "theoretical_loss": 3.552839422915956, + "tokens_seen": 1334837248 + }, + { + "epoch": 0.4, + "learning_rate": 0.000300802407221665, + "loss": 2.905, + "theoretical_loss": 3.552823697549562, + "tokens_seen": 1334902784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007923771313942, + "loss": 2.7399, + "theoretical_loss": 3.552807973171328, + "tokens_seen": 1334968320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007823470411234, + "loss": 2.8187, + "theoretical_loss": 3.552792249781144, + "tokens_seen": 1335033856 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030077231695085254, + "loss": 2.8911, + "theoretical_loss": 3.5527765273788985, + "tokens_seen": 1335099392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007622868605818, + "loss": 2.7133, + "theoretical_loss": 3.552760805964482, + "tokens_seen": 1335164928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007522567703109, + "loss": 2.7799, + "theoretical_loss": 3.552745085537783, + "tokens_seen": 1335230464 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 736883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6608688831329346, + "objective/train/theoretical_loss": 3.552729366098692, + "objective/train/tokens_used": 1355756000, + "theoretical_loss": 3.552729366098692, + "tokens_seen": 1335296000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030074222668004014, + "loss": 2.7968, + "theoretical_loss": 3.552729366098692, + "tokens_seen": 1335296000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007321965897693, + "loss": 2.9199, + "theoretical_loss": 3.5527136476470975, + "tokens_seen": 1335361536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007221664994985, + "loss": 2.7619, + "theoretical_loss": 3.55269793018289, + "tokens_seen": 1335427072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007121364092277, + "loss": 2.5371, + "theoretical_loss": 3.5526822137059577, + "tokens_seen": 1335492608 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007021063189569, + "loss": 2.8803, + "theoretical_loss": 3.5526664982161913, + "tokens_seen": 1335558144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030069207622868604, + "loss": 2.7017, + "theoretical_loss": 3.5526507837134798, + "tokens_seen": 1335623680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006820461384153, + "loss": 2.7533, + "theoretical_loss": 3.5526350701977134, + "tokens_seen": 1335689216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006720160481444, + "loss": 2.6957, + "theoretical_loss": 3.552619357668781, + "tokens_seen": 1335754752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030066198595787364, + "loss": 2.6309, + "theoretical_loss": 3.552603646126572, + "tokens_seen": 1335820288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006519558676028, + "loss": 2.7543, + "theoretical_loss": 3.552587935570977, + "tokens_seen": 1335885824 + }, + { + "epoch": 0.4, + "learning_rate": 0.000300641925777332, + "loss": 2.8341, + "theoretical_loss": 3.5525722260018853, + "tokens_seen": 1335951360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006318956870612, + "loss": 2.8162, + "theoretical_loss": 3.5525565174191858, + "tokens_seen": 1336016896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030062186559679036, + "loss": 2.8456, + "theoretical_loss": 3.5525408098227693, + "tokens_seen": 1336082432 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030061183550651955, + "loss": 2.6389, + "theoretical_loss": 3.552525103212525, + "tokens_seen": 1336147968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006018054162488, + "loss": 2.6604, + "theoretical_loss": 3.552509397588342, + "tokens_seen": 1336213504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003005917753259779, + "loss": 2.9117, + "theoretical_loss": 3.5524936929501107, + "tokens_seen": 1336279040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030058174523570714, + "loss": 2.623, + "theoretical_loss": 3.5524779892977216, + "tokens_seen": 1336344576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030057171514543627, + "loss": 2.7888, + "theoretical_loss": 3.552462286631063, + "tokens_seen": 1336410112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003005616850551655, + "loss": 2.7198, + "theoretical_loss": 3.5524465849500255, + "tokens_seen": 1336475648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003005516549648947, + "loss": 2.7033, + "theoretical_loss": 3.552430884254499, + "tokens_seen": 1336541184 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030054162487462387, + "loss": 2.4967, + "theoretical_loss": 3.5524151845443726, + "tokens_seen": 1336606720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030053159478435305, + "loss": 2.4972, + "theoretical_loss": 3.552399485819537, + "tokens_seen": 1336672256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003005215646940823, + "loss": 2.824, + "theoretical_loss": 3.552383788079881, + "tokens_seen": 1336737792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003005115346038114, + "loss": 2.9612, + "theoretical_loss": 3.552368091325296, + "tokens_seen": 1336803328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030050150451354065, + "loss": 2.6908, + "theoretical_loss": 3.552352395555671, + "tokens_seen": 1336868864 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 737387, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.81619930267334, + "objective/train/theoretical_loss": 3.552336700770896, + "objective/train/tokens_used": 1357394400, + "theoretical_loss": 3.552336700770896, + "tokens_seen": 1336934400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004914744232698, + "loss": 2.7363, + "theoretical_loss": 3.552336700770896, + "tokens_seen": 1336934400 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300481444332999, + "loss": 2.797, + "theoretical_loss": 3.5523210069708604, + "tokens_seen": 1336999936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004714142427282, + "loss": 2.6272, + "theoretical_loss": 3.552305314155455, + "tokens_seen": 1337065472 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030046138415245737, + "loss": 2.8724, + "theoretical_loss": 3.5522896223245697, + "tokens_seen": 1337131008 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030045135406218655, + "loss": 2.7165, + "theoretical_loss": 3.5522739314780942, + "tokens_seen": 1337196544 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030044132397191573, + "loss": 2.9079, + "theoretical_loss": 3.5522582416159176, + "tokens_seen": 1337262080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004312938816449, + "loss": 2.7132, + "theoretical_loss": 3.552242552737932, + "tokens_seen": 1337327616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030042126379137415, + "loss": 2.9322, + "theoretical_loss": 3.552226864844026, + "tokens_seen": 1337393152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004112337011033, + "loss": 2.8038, + "theoretical_loss": 3.5522111779340895, + "tokens_seen": 1337458688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004012036108325, + "loss": 2.7096, + "theoretical_loss": 3.552195492008013, + "tokens_seen": 1337524224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003911735205617, + "loss": 2.8169, + "theoretical_loss": 3.552179807065687, + "tokens_seen": 1337589760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003811434302909, + "loss": 2.6221, + "theoretical_loss": 3.5521641231070014, + "tokens_seen": 1337655296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030037111334002006, + "loss": 2.737, + "theoretical_loss": 3.5521484401318455, + "tokens_seen": 1337720832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030036108324974924, + "loss": 2.7672, + "theoretical_loss": 3.55213275814011, + "tokens_seen": 1337786368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003510531594784, + "loss": 2.9049, + "theoretical_loss": 3.552117077131686, + "tokens_seen": 1337851904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030034102306920765, + "loss": 2.7507, + "theoretical_loss": 3.552101397106462, + "tokens_seen": 1337917440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003309929789368, + "loss": 2.6721, + "theoretical_loss": 3.5520857180643297, + "tokens_seen": 1337982976 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300320962888666, + "loss": 2.6487, + "theoretical_loss": 3.5520700400051783, + "tokens_seen": 1338048512 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030031093279839514, + "loss": 2.6749, + "theoretical_loss": 3.5520543629288976, + "tokens_seen": 1338114048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003009027081244, + "loss": 2.859, + "theoretical_loss": 3.55203868683538, + "tokens_seen": 1338179584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002908726178536, + "loss": 2.8069, + "theoretical_loss": 3.5520230117245135, + "tokens_seen": 1338245120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030028084252758274, + "loss": 2.6247, + "theoretical_loss": 3.5520073375961894, + "tokens_seen": 1338310656 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300270812437312, + "loss": 2.5632, + "theoretical_loss": 3.551991664450298, + "tokens_seen": 1338376192 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002607823470411, + "loss": 2.7317, + "theoretical_loss": 3.5519759922867298, + "tokens_seen": 1338441728 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030025075225677034, + "loss": 2.8898, + "theoretical_loss": 3.5519603211053745, + "tokens_seen": 1338507264 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 738438, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9319119453430176, + "objective/train/theoretical_loss": 3.5519446509061225, + "objective/train/tokens_used": 1359032800, + "theoretical_loss": 3.5519446509061225, + "tokens_seen": 1338572800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002407221664995, + "loss": 2.7252, + "theoretical_loss": 3.5519446509061225, + "tokens_seen": 1338572800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002306920762287, + "loss": 2.7483, + "theoretical_loss": 3.5519289816888646, + "tokens_seen": 1338638336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002206619859579, + "loss": 2.7988, + "theoretical_loss": 3.5519133134534915, + "tokens_seen": 1338703872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002106318956871, + "loss": 2.601, + "theoretical_loss": 3.5518976461998926, + "tokens_seen": 1338769408 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030020060180541624, + "loss": 2.6018, + "theoretical_loss": 3.551881979927959, + "tokens_seen": 1338834944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001905717151455, + "loss": 2.8197, + "theoretical_loss": 3.5518663146375813, + "tokens_seen": 1338900480 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001805416248746, + "loss": 2.6897, + "theoretical_loss": 3.5518506503286496, + "tokens_seen": 1338966016 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030017051153460384, + "loss": 2.6883, + "theoretical_loss": 3.551834987001054, + "tokens_seen": 1339031552 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300160481444333, + "loss": 2.759, + "theoretical_loss": 3.5518193246546863, + "tokens_seen": 1339097088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001504513540622, + "loss": 2.5986, + "theoretical_loss": 3.5518036632894354, + "tokens_seen": 1339162624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001404212637914, + "loss": 2.682, + "theoretical_loss": 3.551788002905193, + "tokens_seen": 1339228160 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030013039117352056, + "loss": 2.6333, + "theoretical_loss": 3.551772343501849, + "tokens_seen": 1339293696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030012036108324975, + "loss": 2.4821, + "theoretical_loss": 3.5517566850792948, + "tokens_seen": 1339359232 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300110330992979, + "loss": 2.5707, + "theoretical_loss": 3.5517410276374197, + "tokens_seen": 1339424768 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001003009027081, + "loss": 2.7059, + "theoretical_loss": 3.5517253711761154, + "tokens_seen": 1339490304 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030009027081243734, + "loss": 2.658, + "theoretical_loss": 3.551709715695272, + "tokens_seen": 1339555840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030008024072216647, + "loss": 2.5603, + "theoretical_loss": 3.5516940611947803, + "tokens_seen": 1339621376 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000702106318957, + "loss": 2.7902, + "theoretical_loss": 3.5516784076745314, + "tokens_seen": 1339686912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000601805416249, + "loss": 2.5795, + "theoretical_loss": 3.551662755134415, + "tokens_seen": 1339752448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030005015045135407, + "loss": 2.6337, + "theoretical_loss": 3.5516471035743225, + "tokens_seen": 1339817984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030004012036108325, + "loss": 2.6817, + "theoretical_loss": 3.551631452994144, + "tokens_seen": 1339883520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000300902708125, + "loss": 2.6522, + "theoretical_loss": 3.5516158033937706, + "tokens_seen": 1339949056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000200601805416, + "loss": 2.9809, + "theoretical_loss": 3.5516001547730935, + "tokens_seen": 1340014592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030001003009027085, + "loss": 2.7736, + "theoretical_loss": 3.5515845071320027, + "tokens_seen": 1340080128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003, + "loss": 2.7429, + "theoretical_loss": 3.551568860470389, + "tokens_seen": 1340145664 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 739370, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.132230043411255, + "objective/train/theoretical_loss": 3.5515532147881443, + "objective/train/tokens_used": 1360671200, + "theoretical_loss": 3.5515532147881443, + "tokens_seen": 1340211200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999899699097292, + "loss": 2.9083, + "theoretical_loss": 3.5515532147881443, + "tokens_seen": 1340211200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999799398194584, + "loss": 2.6438, + "theoretical_loss": 3.5515375700851584, + "tokens_seen": 1340276736 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029996990972918757, + "loss": 2.7103, + "theoretical_loss": 3.551521926361322, + "tokens_seen": 1340342272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029995987963891675, + "loss": 2.5785, + "theoretical_loss": 3.551506283616526, + "tokens_seen": 1340407808 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029994984954864593, + "loss": 2.816, + "theoretical_loss": 3.551490641850662, + "tokens_seen": 1340473344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999398194583751, + "loss": 2.5375, + "theoretical_loss": 3.5514750010636202, + "tokens_seen": 1340538880 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029992978936810435, + "loss": 2.6688, + "theoretical_loss": 3.551459361255292, + "tokens_seen": 1340604416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999197592778335, + "loss": 2.9732, + "theoretical_loss": 3.551443722425568, + "tokens_seen": 1340669952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999097291875627, + "loss": 2.5551, + "theoretical_loss": 3.551428084574339, + "tokens_seen": 1340735488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998996990972919, + "loss": 2.9285, + "theoretical_loss": 3.5514124477014963, + "tokens_seen": 1340801024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998896690070211, + "loss": 2.672, + "theoretical_loss": 3.5513968118069306, + "tokens_seen": 1340866560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029987963891675026, + "loss": 2.6653, + "theoretical_loss": 3.5513811768905335, + "tokens_seen": 1340932096 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029986960882647944, + "loss": 2.644, + "theoretical_loss": 3.551365542952195, + "tokens_seen": 1340997632 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998595787362086, + "loss": 2.7729, + "theoretical_loss": 3.5513499099918064, + "tokens_seen": 1341063168 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029984954864593785, + "loss": 2.9094, + "theoretical_loss": 3.551334278009259, + "tokens_seen": 1341128704 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299839518555667, + "loss": 2.8276, + "theoretical_loss": 3.551318647004444, + "tokens_seen": 1341194240 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998294884653962, + "loss": 2.7759, + "theoretical_loss": 3.5513030169772524, + "tokens_seen": 1341259776 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029981945837512534, + "loss": 2.7611, + "theoretical_loss": 3.551287387927575, + "tokens_seen": 1341325312 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998094282848546, + "loss": 2.952, + "theoretical_loss": 3.551271759855303, + "tokens_seen": 1341390848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029979939819458376, + "loss": 2.5767, + "theoretical_loss": 3.5512561327603276, + "tokens_seen": 1341456384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029978936810431294, + "loss": 2.703, + "theoretical_loss": 3.5512405066425403, + "tokens_seen": 1341521920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997793380140421, + "loss": 2.6325, + "theoretical_loss": 3.551224881501832, + "tokens_seen": 1341587456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997693079237713, + "loss": 2.702, + "theoretical_loss": 3.551209257338093, + "tokens_seen": 1341652992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997592778335005, + "loss": 2.6736, + "theoretical_loss": 3.5511936341512156, + "tokens_seen": 1341718528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997492477432297, + "loss": 2.6862, + "theoretical_loss": 3.551178011941091, + "tokens_seen": 1341784064 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 740011, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.746211528778076, + "objective/train/theoretical_loss": 3.55116239070761, + "objective/train/tokens_used": 1362309600, + "theoretical_loss": 3.55116239070761, + "tokens_seen": 1341849600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029973921765295885, + "loss": 2.5824, + "theoretical_loss": 3.55116239070761, + "tokens_seen": 1341849600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997291875626881, + "loss": 2.8266, + "theoretical_loss": 3.5511467704506643, + "tokens_seen": 1341915136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029971915747241726, + "loss": 2.6679, + "theoretical_loss": 3.5511311511701447, + "tokens_seen": 1341980672 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029970912738214644, + "loss": 2.541, + "theoretical_loss": 3.5511155328659427, + "tokens_seen": 1342046208 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996990972918756, + "loss": 2.7126, + "theoretical_loss": 3.5510999155379492, + "tokens_seen": 1342111744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996890672016048, + "loss": 2.6137, + "theoretical_loss": 3.551084299186056, + "tokens_seen": 1342177280 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299679037111334, + "loss": 2.5994, + "theoretical_loss": 3.551068683810154, + "tokens_seen": 1342242816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996690070210632, + "loss": 2.7638, + "theoretical_loss": 3.5510530694101354, + "tokens_seen": 1342308352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029965897693079235, + "loss": 2.6661, + "theoretical_loss": 3.551037455985891, + "tokens_seen": 1342373888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996489468405216, + "loss": 2.9079, + "theoretical_loss": 3.5510218435373115, + "tokens_seen": 1342439424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996389167502507, + "loss": 2.7773, + "theoretical_loss": 3.5510062320642897, + "tokens_seen": 1342504960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029962888665997995, + "loss": 2.8018, + "theoretical_loss": 3.550990621566716, + "tokens_seen": 1342570496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029961885656970913, + "loss": 2.7557, + "theoretical_loss": 3.550975012044482, + "tokens_seen": 1342636032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996088264794383, + "loss": 2.7851, + "theoretical_loss": 3.55095940349748, + "tokens_seen": 1342701568 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995987963891675, + "loss": 2.8343, + "theoretical_loss": 3.5509437959256003, + "tokens_seen": 1342767104 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029958876629889667, + "loss": 2.6933, + "theoretical_loss": 3.550928189328735, + "tokens_seen": 1342832640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029957873620862585, + "loss": 2.8134, + "theoretical_loss": 3.5509125837067756, + "tokens_seen": 1342898176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995687061183551, + "loss": 2.566, + "theoretical_loss": 3.5508969790596137, + "tokens_seen": 1342963712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995586760280842, + "loss": 2.6247, + "theoretical_loss": 3.5508813753871404, + "tokens_seen": 1343029248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029954864593781345, + "loss": 2.6155, + "theoretical_loss": 3.5508657726892476, + "tokens_seen": 1343094784 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995386158475427, + "loss": 2.5773, + "theoretical_loss": 3.550850170965827, + "tokens_seen": 1343160320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995285857572718, + "loss": 2.9044, + "theoretical_loss": 3.55083457021677, + "tokens_seen": 1343225856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029951855566700105, + "loss": 2.5965, + "theoretical_loss": 3.550818970441968, + "tokens_seen": 1343291392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995085255767302, + "loss": 2.7735, + "theoretical_loss": 3.5508033716413125, + "tokens_seen": 1343356928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994984954864594, + "loss": 2.6961, + "theoretical_loss": 3.5507877738146965, + "tokens_seen": 1343422464 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 740980, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.549888849258423, + "objective/train/theoretical_loss": 3.5507721769620098, + "objective/train/tokens_used": 1363948000, + "theoretical_loss": 3.5507721769620098, + "tokens_seen": 1343488000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994884653961886, + "loss": 2.7331, + "theoretical_loss": 3.5507721769620098, + "tokens_seen": 1343488000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029947843530591777, + "loss": 2.835, + "theoretical_loss": 3.5507565810831454, + "tokens_seen": 1343553536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029946840521564695, + "loss": 2.7311, + "theoretical_loss": 3.5507409861779946, + "tokens_seen": 1343619072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029945837512537613, + "loss": 2.8276, + "theoretical_loss": 3.5507253922464486, + "tokens_seen": 1343684608 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994483450351053, + "loss": 2.7563, + "theoretical_loss": 3.5507097992884002, + "tokens_seen": 1343750144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029943831494483455, + "loss": 2.7471, + "theoretical_loss": 3.5506942073037404, + "tokens_seen": 1343815680 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994282848545637, + "loss": 2.637, + "theoretical_loss": 3.550678616292361, + "tokens_seen": 1343881216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994182547642929, + "loss": 2.6875, + "theoretical_loss": 3.550663026254154, + "tokens_seen": 1343946752 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994082246740221, + "loss": 2.6548, + "theoretical_loss": 3.5506474371890113, + "tokens_seen": 1344012288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993981945837513, + "loss": 2.7173, + "theoretical_loss": 3.5506318490968245, + "tokens_seen": 1344077824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029938816449348046, + "loss": 2.759, + "theoretical_loss": 3.5506162619774857, + "tokens_seen": 1344143360 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029937813440320964, + "loss": 2.7381, + "theoretical_loss": 3.5506006758308857, + "tokens_seen": 1344208896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993681043129388, + "loss": 2.7894, + "theoretical_loss": 3.5505850906569183, + "tokens_seen": 1344274432 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029935807422266805, + "loss": 2.6716, + "theoretical_loss": 3.5505695064554734, + "tokens_seen": 1344339968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993480441323972, + "loss": 2.7826, + "theoretical_loss": 3.5505539232264445, + "tokens_seen": 1344405504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993380140421264, + "loss": 2.8918, + "theoretical_loss": 3.550538340969722, + "tokens_seen": 1344471040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029932798395185554, + "loss": 2.6461, + "theoretical_loss": 3.5505227596851996, + "tokens_seen": 1344536576 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993179538615848, + "loss": 2.8538, + "theoretical_loss": 3.550507179372768, + "tokens_seen": 1344602112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029930792377131396, + "loss": 2.5681, + "theoretical_loss": 3.5504916000323195, + "tokens_seen": 1344667648 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029929789368104314, + "loss": 2.5183, + "theoretical_loss": 3.550476021663746, + "tokens_seen": 1344733184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992878635907723, + "loss": 2.8899, + "theoretical_loss": 3.55046044426694, + "tokens_seen": 1344798720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992778335005015, + "loss": 2.8328, + "theoretical_loss": 3.550444867841793, + "tokens_seen": 1344864256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992678034102307, + "loss": 2.7164, + "theoretical_loss": 3.550429292388197, + "tokens_seen": 1344929792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992577733199599, + "loss": 2.7924, + "theoretical_loss": 3.5504137179060447, + "tokens_seen": 1344995328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029924774322968905, + "loss": 2.8093, + "theoretical_loss": 3.5503981443952273, + "tokens_seen": 1345060864 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 742066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.678600311279297, + "objective/train/theoretical_loss": 3.5503825718556374, + "objective/train/tokens_used": 1365586400, + "theoretical_loss": 3.5503825718556374, + "tokens_seen": 1345126400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992377131394183, + "loss": 2.6478, + "theoretical_loss": 3.5503825718556374, + "tokens_seen": 1345126400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029922768304914746, + "loss": 2.9484, + "theoretical_loss": 3.5503670002871672, + "tokens_seen": 1345191936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029921765295887664, + "loss": 2.7338, + "theoretical_loss": 3.5503514296897087, + "tokens_seen": 1345257472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992076228686058, + "loss": 2.8106, + "theoretical_loss": 3.550335860063154, + "tokens_seen": 1345323008 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299197592778335, + "loss": 2.5024, + "theoretical_loss": 3.5503202914073952, + "tokens_seen": 1345388544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991875626880642, + "loss": 2.7086, + "theoretical_loss": 3.5503047237223253, + "tokens_seen": 1345454080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991775325977934, + "loss": 2.5144, + "theoretical_loss": 3.5502891570078354, + "tokens_seen": 1345519616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029916750250752255, + "loss": 2.7738, + "theoretical_loss": 3.5502735912638177, + "tokens_seen": 1345585152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991574724172518, + "loss": 2.7315, + "theoretical_loss": 3.550258026490165, + "tokens_seen": 1345650688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991474423269809, + "loss": 2.6275, + "theoretical_loss": 3.5502424626867697, + "tokens_seen": 1345716224 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029913741223671015, + "loss": 2.2516, + "theoretical_loss": 3.5502268998535236, + "tokens_seen": 1345781760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029912738214643933, + "loss": 2.7417, + "theoretical_loss": 3.5502113379903193, + "tokens_seen": 1345847296 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991173520561685, + "loss": 2.6809, + "theoretical_loss": 3.5501957770970494, + "tokens_seen": 1345912832 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991073219658977, + "loss": 2.5831, + "theoretical_loss": 3.550180217173605, + "tokens_seen": 1345978368 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029909729187562687, + "loss": 2.5846, + "theoretical_loss": 3.5501646582198796, + "tokens_seen": 1346043904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029908726178535605, + "loss": 2.4565, + "theoretical_loss": 3.550149100235765, + "tokens_seen": 1346109440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990772316950853, + "loss": 2.7977, + "theoretical_loss": 3.550133543221154, + "tokens_seen": 1346174976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990672016048144, + "loss": 2.746, + "theoretical_loss": 3.5501179871759385, + "tokens_seen": 1346240512 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029905717151454365, + "loss": 2.8822, + "theoretical_loss": 3.5501024321000116, + "tokens_seen": 1346306048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029904714142427283, + "loss": 2.4634, + "theoretical_loss": 3.550086877993265, + "tokens_seen": 1346371584 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299037111334002, + "loss": 2.799, + "theoretical_loss": 3.5500713248555913, + "tokens_seen": 1346437120 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990270812437312, + "loss": 2.738, + "theoretical_loss": 3.5500557726868838, + "tokens_seen": 1346502656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990170511534604, + "loss": 2.6585, + "theoretical_loss": 3.5500402214870332, + "tokens_seen": 1346568192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029900702106318956, + "loss": 2.7074, + "theoretical_loss": 3.550024671255934, + "tokens_seen": 1346633728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989969909729188, + "loss": 2.8023, + "theoretical_loss": 3.550009121993477, + "tokens_seen": 1346699264 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 742723, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1151106357574463, + "objective/train/theoretical_loss": 3.549993573699556, + "objective/train/tokens_used": 1367224800, + "theoretical_loss": 3.549993573699556, + "tokens_seen": 1346764800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989869608826479, + "loss": 2.9029, + "theoretical_loss": 3.549993573699556, + "tokens_seen": 1346764800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029897693079237715, + "loss": 2.801, + "theoretical_loss": 3.5499780263740632, + "tokens_seen": 1346830336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989669007021063, + "loss": 2.5526, + "theoretical_loss": 3.549962480016891, + "tokens_seen": 1346895872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989568706118355, + "loss": 2.8075, + "theoretical_loss": 3.5499469346279318, + "tokens_seen": 1346961408 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989468405215647, + "loss": 2.8386, + "theoretical_loss": 3.549931390207078, + "tokens_seen": 1347026944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989368104312939, + "loss": 2.6807, + "theoretical_loss": 3.5499158467542227, + "tokens_seen": 1347092480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029892678034102306, + "loss": 2.5482, + "theoretical_loss": 3.5499003042692587, + "tokens_seen": 1347158016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989167502507523, + "loss": 2.6735, + "theoretical_loss": 3.549884762752079, + "tokens_seen": 1347223552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989067201604814, + "loss": 2.716, + "theoretical_loss": 3.5498692222025747, + "tokens_seen": 1347289088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029889669007021066, + "loss": 2.7231, + "theoretical_loss": 3.5498536826206397, + "tokens_seen": 1347354624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988866599799398, + "loss": 2.6743, + "theoretical_loss": 3.5498381440061664, + "tokens_seen": 1347420160 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298876629889669, + "loss": 2.336, + "theoretical_loss": 3.5498226063590477, + "tokens_seen": 1347485696 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988665997993982, + "loss": 2.6778, + "theoretical_loss": 3.549807069679176, + "tokens_seen": 1347551232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988565697091274, + "loss": 2.6835, + "theoretical_loss": 3.5497915339664443, + "tokens_seen": 1347616768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029884653961885656, + "loss": 2.7947, + "theoretical_loss": 3.5497759992207456, + "tokens_seen": 1347682304 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029883650952858574, + "loss": 2.5446, + "theoretical_loss": 3.549760465441972, + "tokens_seen": 1347747840 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988264794383149, + "loss": 2.7514, + "theoretical_loss": 3.549744932630017, + "tokens_seen": 1347813376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029881644934804416, + "loss": 2.7718, + "theoretical_loss": 3.5497294007847726, + "tokens_seen": 1347878912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988064192577733, + "loss": 2.6986, + "theoretical_loss": 3.5497138699061326, + "tokens_seen": 1347944448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987963891675025, + "loss": 2.8023, + "theoretical_loss": 3.5496983399939888, + "tokens_seen": 1348009984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987863590772317, + "loss": 2.6328, + "theoretical_loss": 3.5496828110482355, + "tokens_seen": 1348075520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987763289869609, + "loss": 2.6734, + "theoretical_loss": 3.5496672830687643, + "tokens_seen": 1348141056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987662988966901, + "loss": 2.5762, + "theoretical_loss": 3.549651756055469, + "tokens_seen": 1348206592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029875626880641925, + "loss": 2.7513, + "theoretical_loss": 3.5496362300082414, + "tokens_seen": 1348272128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987462387161485, + "loss": 3.0143, + "theoretical_loss": 3.5496207049269755, + "tokens_seen": 1348337664 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 744066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.647536277770996, + "objective/train/theoretical_loss": 3.5496051808115636, + "objective/train/tokens_used": 1368863200, + "theoretical_loss": 3.5496051808115636, + "tokens_seen": 1348403200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029873620862587766, + "loss": 2.4889, + "theoretical_loss": 3.5496051808115636, + "tokens_seen": 1348403200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029872617853560684, + "loss": 2.4901, + "theoretical_loss": 3.5495896576618993, + "tokens_seen": 1348468736 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298716148445336, + "loss": 2.68, + "theoretical_loss": 3.549574135477875, + "tokens_seen": 1348534272 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987061183550652, + "loss": 2.5606, + "theoretical_loss": 3.549558614259384, + "tokens_seen": 1348599808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986960882647944, + "loss": 2.7406, + "theoretical_loss": 3.549543094006319, + "tokens_seen": 1348665344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986860581745236, + "loss": 2.659, + "theoretical_loss": 3.549527574718574, + "tokens_seen": 1348730880 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029867602808425275, + "loss": 2.9053, + "theoretical_loss": 3.5495120563960407, + "tokens_seen": 1348796416 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298665997993982, + "loss": 2.4369, + "theoretical_loss": 3.5494965390386133, + "tokens_seen": 1348861952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986559679037111, + "loss": 2.6933, + "theoretical_loss": 3.5494810226461846, + "tokens_seen": 1348927488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029864593781344035, + "loss": 2.5889, + "theoretical_loss": 3.549465507218647, + "tokens_seen": 1348993024 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029863590772316953, + "loss": 2.8407, + "theoretical_loss": 3.5494499927558945, + "tokens_seen": 1349058560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986258776328987, + "loss": 2.5919, + "theoretical_loss": 3.54943447925782, + "tokens_seen": 1349124096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986158475426279, + "loss": 2.6487, + "theoretical_loss": 3.549418966724316, + "tokens_seen": 1349189632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029860581745235707, + "loss": 2.9139, + "theoretical_loss": 3.549403455155277, + "tokens_seen": 1349255168 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029859578736208625, + "loss": 2.617, + "theoretical_loss": 3.549387944550595, + "tokens_seen": 1349320704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985857572718155, + "loss": 2.6767, + "theoretical_loss": 3.549372434910164, + "tokens_seen": 1349386240 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985757271815446, + "loss": 2.6744, + "theoretical_loss": 3.549356926233876, + "tokens_seen": 1349451776 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029856569709127385, + "loss": 2.6996, + "theoretical_loss": 3.5493414185216263, + "tokens_seen": 1349517312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029855566700100303, + "loss": 2.6145, + "theoretical_loss": 3.5493259117733063, + "tokens_seen": 1349582848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985456369107322, + "loss": 2.7506, + "theoretical_loss": 3.54931040598881, + "tokens_seen": 1349648384 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985356068204614, + "loss": 2.7514, + "theoretical_loss": 3.549294901168031, + "tokens_seen": 1349713920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985255767301906, + "loss": 2.6993, + "theoretical_loss": 3.549279397310862, + "tokens_seen": 1349779456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029851554663991976, + "loss": 2.4447, + "theoretical_loss": 3.549263894417197, + "tokens_seen": 1349844992 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298505516549649, + "loss": 2.6124, + "theoretical_loss": 3.5492483924869287, + "tokens_seen": 1349910528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984954864593781, + "loss": 2.5441, + "theoretical_loss": 3.5492328915199503, + "tokens_seen": 1349976064 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 744638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0549464225769043, + "objective/train/theoretical_loss": 3.5492173915161565, + "objective/train/tokens_used": 1370501600, + "theoretical_loss": 3.5492173915161565, + "tokens_seen": 1350041600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029848545636910735, + "loss": 3.036, + "theoretical_loss": 3.5492173915161565, + "tokens_seen": 1350041600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984754262788365, + "loss": 2.7784, + "theoretical_loss": 3.5492018924754394, + "tokens_seen": 1350107136 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984653961885657, + "loss": 2.6132, + "theoretical_loss": 3.549186394397693, + "tokens_seen": 1350172672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984553660982949, + "loss": 2.9592, + "theoretical_loss": 3.54917089728281, + "tokens_seen": 1350238208 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984453360080241, + "loss": 2.4644, + "theoretical_loss": 3.549155401130685, + "tokens_seen": 1350303744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029843530591775326, + "loss": 2.7715, + "theoretical_loss": 3.5491399059412103, + "tokens_seen": 1350369280 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984252758274825, + "loss": 2.7839, + "theoretical_loss": 3.5491244117142804, + "tokens_seen": 1350434816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984152457372116, + "loss": 2.6439, + "theoretical_loss": 3.549108918449788, + "tokens_seen": 1350500352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029840521564694086, + "loss": 2.6755, + "theoretical_loss": 3.5490934261476275, + "tokens_seen": 1350565888 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029839518555667, + "loss": 2.6529, + "theoretical_loss": 3.5490779348076917, + "tokens_seen": 1350631424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983851554663992, + "loss": 2.3959, + "theoretical_loss": 3.5490624444298744, + "tokens_seen": 1350696960 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983751253761284, + "loss": 2.5926, + "theoretical_loss": 3.5490469550140693, + "tokens_seen": 1350762496 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983650952858576, + "loss": 2.8891, + "theoretical_loss": 3.5490314665601694, + "tokens_seen": 1350828032 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029835506519558676, + "loss": 2.5595, + "theoretical_loss": 3.5490159790680686, + "tokens_seen": 1350893568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029834503510531594, + "loss": 2.7735, + "theoretical_loss": 3.549000492537661, + "tokens_seen": 1350959104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983350050150451, + "loss": 2.6252, + "theoretical_loss": 3.54898500696884, + "tokens_seen": 1351024640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029832497492477436, + "loss": 2.6226, + "theoretical_loss": 3.5489695223614985, + "tokens_seen": 1351090176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983149448345035, + "loss": 2.7547, + "theoretical_loss": 3.5489540387155314, + "tokens_seen": 1351155712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983049147442327, + "loss": 2.6143, + "theoretical_loss": 3.5489385560308317, + "tokens_seen": 1351221248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029829488465396185, + "loss": 2.5784, + "theoretical_loss": 3.548923074307293, + "tokens_seen": 1351286784 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982848545636911, + "loss": 2.8404, + "theoretical_loss": 3.548907593544809, + "tokens_seen": 1351352320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029827482447342026, + "loss": 2.618, + "theoretical_loss": 3.548892113743274, + "tokens_seen": 1351417856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029826479438314945, + "loss": 2.8315, + "theoretical_loss": 3.548876634902581, + "tokens_seen": 1351483392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982547642928786, + "loss": 2.9895, + "theoretical_loss": 3.5488611570226247, + "tokens_seen": 1351548928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029824473420260786, + "loss": 2.4631, + "theoretical_loss": 3.548845680103298, + "tokens_seen": 1351614464 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 745869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0698626041412354, + "objective/train/theoretical_loss": 3.5488302041444952, + "objective/train/tokens_used": 1372140000, + "theoretical_loss": 3.5488302041444952, + "tokens_seen": 1351680000 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298234704112337, + "loss": 2.8399, + "theoretical_loss": 3.5488302041444952, + "tokens_seen": 1351680000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982246740220662, + "loss": 2.6906, + "theoretical_loss": 3.54881472914611, + "tokens_seen": 1351745536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029821464393179535, + "loss": 2.7309, + "theoretical_loss": 3.5487992551080363, + "tokens_seen": 1351811072 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982046138415246, + "loss": 2.728, + "theoretical_loss": 3.548783782030168, + "tokens_seen": 1351876608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029819458375125377, + "loss": 2.6026, + "theoretical_loss": 3.5487683099123983, + "tokens_seen": 1351942144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029818455366098295, + "loss": 2.4077, + "theoretical_loss": 3.5487528387546217, + "tokens_seen": 1352007680 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029817452357071213, + "loss": 2.8385, + "theoretical_loss": 3.5487373685567327, + "tokens_seen": 1352073216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981644934804413, + "loss": 2.7771, + "theoretical_loss": 3.548721899318624, + "tokens_seen": 1352138752 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981544633901705, + "loss": 2.7378, + "theoretical_loss": 3.5487064310401903, + "tokens_seen": 1352204288 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029814443329989973, + "loss": 2.6337, + "theoretical_loss": 3.5486909637213255, + "tokens_seen": 1352269824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029813440320962885, + "loss": 2.7281, + "theoretical_loss": 3.548675497361923, + "tokens_seen": 1352335360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981243731193581, + "loss": 2.8302, + "theoretical_loss": 3.5486600319618775, + "tokens_seen": 1352400896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981143430290872, + "loss": 2.7914, + "theoretical_loss": 3.548644567521083, + "tokens_seen": 1352466432 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029810431293881645, + "loss": 2.9081, + "theoretical_loss": 3.5486291040394327, + "tokens_seen": 1352531968 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029809428284854563, + "loss": 2.7718, + "theoretical_loss": 3.5486136415168215, + "tokens_seen": 1352597504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980842527582748, + "loss": 2.6835, + "theoretical_loss": 3.548598179953143, + "tokens_seen": 1352663040 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298074222668004, + "loss": 2.7298, + "theoretical_loss": 3.5485827193482917, + "tokens_seen": 1352728576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029806419257773323, + "loss": 2.573, + "theoretical_loss": 3.5485672597021614, + "tokens_seen": 1352794112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029805416248746236, + "loss": 2.6888, + "theoretical_loss": 3.548551801014646, + "tokens_seen": 1352859648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980441323971916, + "loss": 2.6973, + "theoretical_loss": 3.54853634328564, + "tokens_seen": 1352925184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980341023069208, + "loss": 2.5712, + "theoretical_loss": 3.5485208865150373, + "tokens_seen": 1352990720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029802407221664996, + "loss": 2.6709, + "theoretical_loss": 3.548505430702732, + "tokens_seen": 1353056256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980140421263792, + "loss": 2.4174, + "theoretical_loss": 3.548489975848619, + "tokens_seen": 1353121792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980040120361083, + "loss": 2.7311, + "theoretical_loss": 3.548474521952591, + "tokens_seen": 1353187328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029799398194583755, + "loss": 2.6042, + "theoretical_loss": 3.548459069014544, + "tokens_seen": 1353252864 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 746951, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.679593801498413, + "objective/train/theoretical_loss": 3.548443617034371, + "objective/train/tokens_used": 1373778400, + "theoretical_loss": 3.548443617034371, + "tokens_seen": 1353318400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979839518555667, + "loss": 2.6849, + "theoretical_loss": 3.548443617034371, + "tokens_seen": 1353318400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979739217652959, + "loss": 2.6242, + "theoretical_loss": 3.5484281660119668, + "tokens_seen": 1353383936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979638916750251, + "loss": 2.6791, + "theoretical_loss": 3.5484127159472254, + "tokens_seen": 1353449472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979538615847543, + "loss": 2.691, + "theoretical_loss": 3.548397266840041, + "tokens_seen": 1353515008 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029794383149448346, + "loss": 2.6914, + "theoretical_loss": 3.5483818186903084, + "tokens_seen": 1353580544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979338014042127, + "loss": 2.5739, + "theoretical_loss": 3.548366371497921, + "tokens_seen": 1353646080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979237713139418, + "loss": 2.7162, + "theoretical_loss": 3.548350925262774, + "tokens_seen": 1353711616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029791374122367106, + "loss": 2.707, + "theoretical_loss": 3.548335479984761, + "tokens_seen": 1353777152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979037111334002, + "loss": 2.5863, + "theoretical_loss": 3.548320035663777, + "tokens_seen": 1353842688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978936810431294, + "loss": 2.7147, + "theoretical_loss": 3.5483045922997167, + "tokens_seen": 1353908224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978836509528586, + "loss": 2.7314, + "theoretical_loss": 3.5482891498924736, + "tokens_seen": 1353973760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978736208625878, + "loss": 2.7817, + "theoretical_loss": 3.5482737084419425, + "tokens_seen": 1354039296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029786359077231696, + "loss": 2.7877, + "theoretical_loss": 3.5482582679480172, + "tokens_seen": 1354104832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029785356068204614, + "loss": 2.6604, + "theoretical_loss": 3.5482428284105936, + "tokens_seen": 1354170368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978435305917753, + "loss": 2.6569, + "theoretical_loss": 3.5482273898295644, + "tokens_seen": 1354235904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029783350050150456, + "loss": 2.4882, + "theoretical_loss": 3.5482119522048254, + "tokens_seen": 1354301440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978234704112337, + "loss": 2.8416, + "theoretical_loss": 3.5481965155362705, + "tokens_seen": 1354366976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978134403209629, + "loss": 2.6932, + "theoretical_loss": 3.548181079823795, + "tokens_seen": 1354432512 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029780341023069205, + "loss": 2.7576, + "theoretical_loss": 3.548165645067292, + "tokens_seen": 1354498048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977933801404213, + "loss": 2.603, + "theoretical_loss": 3.548150211266657, + "tokens_seen": 1354563584 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029778335005015046, + "loss": 2.6103, + "theoretical_loss": 3.548134778421785, + "tokens_seen": 1354629120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029777331995987965, + "loss": 2.5713, + "theoretical_loss": 3.5481193465325696, + "tokens_seen": 1354694656 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029776328986960883, + "loss": 2.6096, + "theoretical_loss": 3.5481039155989054, + "tokens_seen": 1354760192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029775325977933806, + "loss": 2.5663, + "theoretical_loss": 3.548088485620688, + "tokens_seen": 1354825728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977432296890672, + "loss": 2.5378, + "theoretical_loss": 3.5480730565978114, + "tokens_seen": 1354891264 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 747636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4359793663024902, + "objective/train/theoretical_loss": 3.54805762853017, + "objective/train/tokens_used": 1375416800, + "theoretical_loss": 3.54805762853017, + "tokens_seen": 1354956800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977331995987964, + "loss": 2.5447, + "theoretical_loss": 3.54805762853017, + "tokens_seen": 1354956800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029772316950852555, + "loss": 2.6167, + "theoretical_loss": 3.5480422014176582, + "tokens_seen": 1355022336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977131394182548, + "loss": 2.6217, + "theoretical_loss": 3.548026775260172, + "tokens_seen": 1355087872 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029770310932798397, + "loss": 2.7159, + "theoretical_loss": 3.548011350057605, + "tokens_seen": 1355153408 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029769307923771315, + "loss": 2.6077, + "theoretical_loss": 3.547995925809852, + "tokens_seen": 1355218944 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029768304914744233, + "loss": 2.473, + "theoretical_loss": 3.5479805025168085, + "tokens_seen": 1355284480 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976730190571715, + "loss": 2.5297, + "theoretical_loss": 3.5479650801783684, + "tokens_seen": 1355350016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976629889669007, + "loss": 2.7728, + "theoretical_loss": 3.547949658794427, + "tokens_seen": 1355415552 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029765295887662993, + "loss": 2.7018, + "theoretical_loss": 3.547934238364878, + "tokens_seen": 1355481088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029764292878635905, + "loss": 2.7848, + "theoretical_loss": 3.5479188188896176, + "tokens_seen": 1355546624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976328986960883, + "loss": 2.732, + "theoretical_loss": 3.54790340036854, + "tokens_seen": 1355612160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976228686058174, + "loss": 2.5588, + "theoretical_loss": 3.54788798280154, + "tokens_seen": 1355677696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029761283851554665, + "loss": 2.66, + "theoretical_loss": 3.547872566188513, + "tokens_seen": 1355743232 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029760280842527583, + "loss": 2.5881, + "theoretical_loss": 3.5478571505293526, + "tokens_seen": 1355808768 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297592778335005, + "loss": 2.5998, + "theoretical_loss": 3.5478417358239547, + "tokens_seen": 1355874304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975827482447342, + "loss": 2.6738, + "theoretical_loss": 3.547826322072214, + "tokens_seen": 1355939840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029757271815446343, + "loss": 2.7197, + "theoretical_loss": 3.5478109092740255, + "tokens_seen": 1356005376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029756268806419256, + "loss": 2.4993, + "theoretical_loss": 3.547795497429284, + "tokens_seen": 1356070912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975526579739218, + "loss": 2.6149, + "theoretical_loss": 3.547780086537884, + "tokens_seen": 1356136448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975426278836509, + "loss": 2.8272, + "theoretical_loss": 3.547764676599721, + "tokens_seen": 1356201984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029753259779338016, + "loss": 2.5975, + "theoretical_loss": 3.54774926761469, + "tokens_seen": 1356267520 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029752256770310934, + "loss": 2.6422, + "theoretical_loss": 3.547733859582686, + "tokens_seen": 1356333056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975125376128385, + "loss": 2.6204, + "theoretical_loss": 3.547718452503604, + "tokens_seen": 1356398592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975025075225677, + "loss": 2.4732, + "theoretical_loss": 3.5477030463773387, + "tokens_seen": 1356464128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974924774322969, + "loss": 2.6265, + "theoretical_loss": 3.547687641203785, + "tokens_seen": 1356529664 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 748231, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.774550676345825, + "objective/train/theoretical_loss": 3.547672236982839, + "objective/train/tokens_used": 1377055200, + "theoretical_loss": 3.547672236982839, + "tokens_seen": 1356595200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029748244734202606, + "loss": 2.7848, + "theoretical_loss": 3.547672236982839, + "tokens_seen": 1356595200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974724172517553, + "loss": 2.9087, + "theoretical_loss": 3.5476568337143943, + "tokens_seen": 1356660736 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974623871614844, + "loss": 2.6341, + "theoretical_loss": 3.5476414313983473, + "tokens_seen": 1356726272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029745235707121366, + "loss": 2.6638, + "theoretical_loss": 3.5476260300345928, + "tokens_seen": 1356791808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974423269809428, + "loss": 2.7465, + "theoretical_loss": 3.547610629623025, + "tokens_seen": 1356857344 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297432296890672, + "loss": 2.5829, + "theoretical_loss": 3.54759523016354, + "tokens_seen": 1356922880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974222668004012, + "loss": 2.376, + "theoretical_loss": 3.547579831656033, + "tokens_seen": 1356988416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974122367101304, + "loss": 2.7145, + "theoretical_loss": 3.5475644341003987, + "tokens_seen": 1357053952 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029740220661985956, + "loss": 2.6275, + "theoretical_loss": 3.547549037496533, + "tokens_seen": 1357119488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973921765295888, + "loss": 2.4773, + "theoretical_loss": 3.54753364184433, + "tokens_seen": 1357185024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973821464393179, + "loss": 2.6747, + "theoretical_loss": 3.547518247143686, + "tokens_seen": 1357250560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029737211634904716, + "loss": 2.6817, + "theoretical_loss": 3.5475028533944952, + "tokens_seen": 1357316096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973620862587763, + "loss": 2.4743, + "theoretical_loss": 3.547487460596654, + "tokens_seen": 1357381632 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973520561685055, + "loss": 2.5971, + "theoretical_loss": 3.547472068750057, + "tokens_seen": 1357447168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973420260782347, + "loss": 2.6343, + "theoretical_loss": 3.5474566778545995, + "tokens_seen": 1357512704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973319959879639, + "loss": 2.7745, + "theoretical_loss": 3.5474412879101767, + "tokens_seen": 1357578240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029732196589769307, + "loss": 2.6456, + "theoretical_loss": 3.5474258989166847, + "tokens_seen": 1357643776 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029731193580742225, + "loss": 2.6217, + "theoretical_loss": 3.547410510874018, + "tokens_seen": 1357709312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029730190571715143, + "loss": 2.9643, + "theoretical_loss": 3.5473951237820724, + "tokens_seen": 1357774848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029729187562688067, + "loss": 2.7048, + "theoretical_loss": 3.5473797376407425, + "tokens_seen": 1357840384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029728184553660985, + "loss": 2.6319, + "theoretical_loss": 3.547364352449925, + "tokens_seen": 1357905920 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029727181544633903, + "loss": 2.7396, + "theoretical_loss": 3.5473489682095147, + "tokens_seen": 1357971456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029726178535606826, + "loss": 2.5027, + "theoretical_loss": 3.5473335849194068, + "tokens_seen": 1358036992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972517552657974, + "loss": 2.7434, + "theoretical_loss": 3.547318202579497, + "tokens_seen": 1358102528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972417251755266, + "loss": 2.4824, + "theoretical_loss": 3.5473028211896804, + "tokens_seen": 1358168064 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 749499, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2978737354278564, + "objective/train/theoretical_loss": 3.547287440749853, + "objective/train/tokens_used": 1378693600, + "theoretical_loss": 3.547287440749853, + "tokens_seen": 1358233600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029723169508525575, + "loss": 2.4681, + "theoretical_loss": 3.547287440749853, + "tokens_seen": 1358233600 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297221664994985, + "loss": 2.7365, + "theoretical_loss": 3.5472720612599096, + "tokens_seen": 1358299136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029721163490471417, + "loss": 2.7585, + "theoretical_loss": 3.5472566827197465, + "tokens_seen": 1358364672 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029720160481444335, + "loss": 2.6884, + "theoretical_loss": 3.5472413051292593, + "tokens_seen": 1358430208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029719157472417253, + "loss": 2.7544, + "theoretical_loss": 3.5472259284883423, + "tokens_seen": 1358495744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971815446339017, + "loss": 2.4551, + "theoretical_loss": 3.5472105527968925, + "tokens_seen": 1358561280 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971715145436309, + "loss": 2.6856, + "theoretical_loss": 3.5471951780548046, + "tokens_seen": 1358626816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029716148445336013, + "loss": 2.8647, + "theoretical_loss": 3.547179804261974, + "tokens_seen": 1358692352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029715145436308926, + "loss": 2.5976, + "theoretical_loss": 3.5471644314182975, + "tokens_seen": 1358757888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971414242728185, + "loss": 2.6342, + "theoretical_loss": 3.5471490595236697, + "tokens_seen": 1358823424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971313941825476, + "loss": 2.7117, + "theoretical_loss": 3.5471336885779867, + "tokens_seen": 1358888960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029712136409227685, + "loss": 2.6165, + "theoretical_loss": 3.547118318581144, + "tokens_seen": 1358954496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029711133400200603, + "loss": 2.5478, + "theoretical_loss": 3.547102949533037, + "tokens_seen": 1359020032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971013039117352, + "loss": 2.5827, + "theoretical_loss": 3.547087581433561, + "tokens_seen": 1359085568 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970912738214644, + "loss": 2.7006, + "theoretical_loss": 3.5470722142826134, + "tokens_seen": 1359151104 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029708124373119363, + "loss": 2.7409, + "theoretical_loss": 3.547056848080089, + "tokens_seen": 1359216640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029707121364092276, + "loss": 2.9835, + "theoretical_loss": 3.5470414828258825, + "tokens_seen": 1359282176 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297061183550652, + "loss": 2.738, + "theoretical_loss": 3.5470261185198915, + "tokens_seen": 1359347712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970511534603811, + "loss": 2.6083, + "theoretical_loss": 3.54701075516201, + "tokens_seen": 1359413248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029704112337011036, + "loss": 2.6068, + "theoretical_loss": 3.546995392752135, + "tokens_seen": 1359478784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029703109327983954, + "loss": 2.6079, + "theoretical_loss": 3.546980031290162, + "tokens_seen": 1359544320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970210631895687, + "loss": 2.963, + "theoretical_loss": 3.5469646707759868, + "tokens_seen": 1359609856 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970110330992979, + "loss": 2.5476, + "theoretical_loss": 3.546949311209505, + "tokens_seen": 1359675392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970010030090271, + "loss": 2.636, + "theoretical_loss": 3.546933952590613, + "tokens_seen": 1359740928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029699097291875626, + "loss": 2.6404, + "theoretical_loss": 3.546918594919206, + "tokens_seen": 1359806464 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 749946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6572976112365723, + "objective/train/theoretical_loss": 3.5469032381951804, + "objective/train/tokens_used": 1380332000, + "theoretical_loss": 3.5469032381951804, + "tokens_seen": 1359872000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969809428284855, + "loss": 2.4872, + "theoretical_loss": 3.5469032381951804, + "tokens_seen": 1359872000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969709127382146, + "loss": 2.815, + "theoretical_loss": 3.5468878824184324, + "tokens_seen": 1359937536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029696088264794386, + "loss": 2.714, + "theoretical_loss": 3.5468725275888566, + "tokens_seen": 1360003072 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296950852557673, + "loss": 2.6146, + "theoretical_loss": 3.5468571737063503, + "tokens_seen": 1360068608 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969408224674022, + "loss": 2.4896, + "theoretical_loss": 3.546841820770809, + "tokens_seen": 1360134144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969307923771314, + "loss": 2.7668, + "theoretical_loss": 3.546826468782128, + "tokens_seen": 1360199680 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969207622868606, + "loss": 2.5955, + "theoretical_loss": 3.546811117740204, + "tokens_seen": 1360265216 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029691073219658976, + "loss": 2.6055, + "theoretical_loss": 3.5467957676449338, + "tokens_seen": 1360330752 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296900702106319, + "loss": 2.5427, + "theoretical_loss": 3.546780418496212, + "tokens_seen": 1360396288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968906720160481, + "loss": 2.3896, + "theoretical_loss": 3.5467650702939353, + "tokens_seen": 1360461824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029688064192577736, + "loss": 2.7381, + "theoretical_loss": 3.5467497230379994, + "tokens_seen": 1360527360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968706118355065, + "loss": 2.7378, + "theoretical_loss": 3.546734376728301, + "tokens_seen": 1360592896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968605817452357, + "loss": 2.6143, + "theoretical_loss": 3.546719031364735, + "tokens_seen": 1360658432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968505516549649, + "loss": 2.6222, + "theoretical_loss": 3.5467036869471986, + "tokens_seen": 1360723968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968405215646941, + "loss": 2.8916, + "theoretical_loss": 3.546688343475588, + "tokens_seen": 1360789504 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029683049147442327, + "loss": 2.6858, + "theoretical_loss": 3.5466730009497986, + "tokens_seen": 1360855040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029682046138415245, + "loss": 2.7056, + "theoretical_loss": 3.5466576593697265, + "tokens_seen": 1360920576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029681043129388163, + "loss": 2.4382, + "theoretical_loss": 3.546642318735269, + "tokens_seen": 1360986112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029680040120361087, + "loss": 2.6948, + "theoretical_loss": 3.546626979046321, + "tokens_seen": 1361051648 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029679037111334, + "loss": 2.7604, + "theoretical_loss": 3.5466116403027796, + "tokens_seen": 1361117184 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029678034102306923, + "loss": 2.631, + "theoretical_loss": 3.5465963025045406, + "tokens_seen": 1361182720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967703109327984, + "loss": 2.432, + "theoretical_loss": 3.5465809656515006, + "tokens_seen": 1361248256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967602808425276, + "loss": 2.4322, + "theoretical_loss": 3.546565629743555, + "tokens_seen": 1361313792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029675025075225677, + "loss": 2.7459, + "theoretical_loss": 3.5465502947806007, + "tokens_seen": 1361379328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029674022066198595, + "loss": 2.7613, + "theoretical_loss": 3.546534960762534, + "tokens_seen": 1361444864 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 750808, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7870075702667236, + "objective/train/theoretical_loss": 3.5465196276892508, + "objective/train/tokens_used": 1381970400, + "theoretical_loss": 3.5465196276892508, + "tokens_seen": 1361510400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029673019057171513, + "loss": 2.8345, + "theoretical_loss": 3.5465196276892508, + "tokens_seen": 1361510400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029672016048144437, + "loss": 2.4962, + "theoretical_loss": 3.5465042955606485, + "tokens_seen": 1361575936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967101303911735, + "loss": 2.8696, + "theoretical_loss": 3.5464889643766218, + "tokens_seen": 1361641472 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029670010030090273, + "loss": 2.4112, + "theoretical_loss": 3.546473634137068, + "tokens_seen": 1361707008 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029669007021063186, + "loss": 2.5725, + "theoretical_loss": 3.5464583048418836, + "tokens_seen": 1361772544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966800401203611, + "loss": 2.7429, + "theoretical_loss": 3.546442976490965, + "tokens_seen": 1361838080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966700100300903, + "loss": 2.6622, + "theoretical_loss": 3.546427649084208, + "tokens_seen": 1361903616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029665997993981946, + "loss": 2.8366, + "theoretical_loss": 3.546412322621509, + "tokens_seen": 1361969152 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029664994984954864, + "loss": 2.6071, + "theoretical_loss": 3.5463969971027653, + "tokens_seen": 1362034688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966399197592778, + "loss": 2.5876, + "theoretical_loss": 3.5463816725278727, + "tokens_seen": 1362100224 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296629889669007, + "loss": 2.6304, + "theoretical_loss": 3.5463663488967274, + "tokens_seen": 1362165760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029661985957873623, + "loss": 2.8303, + "theoretical_loss": 3.546351026209227, + "tokens_seen": 1362231296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029660982948846536, + "loss": 2.7284, + "theoretical_loss": 3.5463357044652666, + "tokens_seen": 1362296832 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965997993981946, + "loss": 2.8072, + "theoretical_loss": 3.5463203836647437, + "tokens_seen": 1362362368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965897693079238, + "loss": 2.775, + "theoretical_loss": 3.546305063807554, + "tokens_seen": 1362427904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029657973921765296, + "loss": 2.8564, + "theoretical_loss": 3.5462897448935955, + "tokens_seen": 1362493440 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029656970912738214, + "loss": 2.6761, + "theoretical_loss": 3.5462744269227633, + "tokens_seen": 1362558976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965596790371113, + "loss": 2.5418, + "theoretical_loss": 3.5462591098949545, + "tokens_seen": 1362624512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965496489468405, + "loss": 2.6304, + "theoretical_loss": 3.5462437938100653, + "tokens_seen": 1362690048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029653961885656974, + "loss": 2.6772, + "theoretical_loss": 3.5462284786679934, + "tokens_seen": 1362755584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965295887662989, + "loss": 2.753, + "theoretical_loss": 3.546213164468634, + "tokens_seen": 1362821120 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965195586760281, + "loss": 2.7359, + "theoretical_loss": 3.546197851211885, + "tokens_seen": 1362886656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965095285857573, + "loss": 2.565, + "theoretical_loss": 3.546182538897642, + "tokens_seen": 1362952192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029649949849548646, + "loss": 2.7624, + "theoretical_loss": 3.5461672275258027, + "tokens_seen": 1363017728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964894684052157, + "loss": 2.5734, + "theoretical_loss": 3.546151917096263, + "tokens_seen": 1363083264 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 751921, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8145251274108887, + "objective/train/theoretical_loss": 3.5461366076089202, + "objective/train/tokens_used": 1383608800, + "theoretical_loss": 3.5461366076089202, + "tokens_seen": 1363148800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964794383149448, + "loss": 2.8472, + "theoretical_loss": 3.5461366076089202, + "tokens_seen": 1363148800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029646940822467406, + "loss": 2.6509, + "theoretical_loss": 3.54612129906367, + "tokens_seen": 1363214336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964593781344032, + "loss": 2.6087, + "theoretical_loss": 3.5461059914604105, + "tokens_seen": 1363279872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964493480441324, + "loss": 2.5793, + "theoretical_loss": 3.5460906847990374, + "tokens_seen": 1363345408 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964393179538616, + "loss": 2.6705, + "theoretical_loss": 3.5460753790794484, + "tokens_seen": 1363410944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964292878635908, + "loss": 2.7595, + "theoretical_loss": 3.54606007430154, + "tokens_seen": 1363476480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029641925777331996, + "loss": 2.6003, + "theoretical_loss": 3.5460447704652074, + "tokens_seen": 1363542016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964092276830492, + "loss": 2.6074, + "theoretical_loss": 3.54602946757035, + "tokens_seen": 1363607552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963991975927783, + "loss": 2.6677, + "theoretical_loss": 3.546014165616863, + "tokens_seen": 1363673088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029638916750250756, + "loss": 2.4795, + "theoretical_loss": 3.545998864604644, + "tokens_seen": 1363738624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963791374122367, + "loss": 2.4361, + "theoretical_loss": 3.5459835645335893, + "tokens_seen": 1363804160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963691073219659, + "loss": 2.8123, + "theoretical_loss": 3.5459682654035958, + "tokens_seen": 1363869696 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963590772316951, + "loss": 2.9173, + "theoretical_loss": 3.5459529672145607, + "tokens_seen": 1363935232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963490471414243, + "loss": 2.4875, + "theoretical_loss": 3.5459376699663814, + "tokens_seen": 1364000768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029633901705115347, + "loss": 2.7815, + "theoretical_loss": 3.5459223736589536, + "tokens_seen": 1364066304 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029632898696088265, + "loss": 2.6226, + "theoretical_loss": 3.5459070782921756, + "tokens_seen": 1364131840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029631895687061183, + "loss": 2.4766, + "theoretical_loss": 3.545891783865944, + "tokens_seen": 1364197376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029630892678034107, + "loss": 2.5906, + "theoretical_loss": 3.5458764903801545, + "tokens_seen": 1364262912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962988966900702, + "loss": 2.5496, + "theoretical_loss": 3.5458611978347054, + "tokens_seen": 1364328448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029628886659979943, + "loss": 2.4896, + "theoretical_loss": 3.545845906229494, + "tokens_seen": 1364393984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962788365095286, + "loss": 2.6595, + "theoretical_loss": 3.545830615564416, + "tokens_seen": 1364459520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962688064192578, + "loss": 2.523, + "theoretical_loss": 3.54581532583937, + "tokens_seen": 1364525056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029625877632898697, + "loss": 2.6689, + "theoretical_loss": 3.545800037054251, + "tokens_seen": 1364590592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029624874623871615, + "loss": 2.4986, + "theoretical_loss": 3.5457847492089587, + "tokens_seen": 1364656128 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029623871614844533, + "loss": 2.4405, + "theoretical_loss": 3.5457694623033884, + "tokens_seen": 1364721664 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 752367, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4154515266418457, + "objective/train/theoretical_loss": 3.5457541763374376, + "objective/train/tokens_used": 1385247200, + "theoretical_loss": 3.5457541763374376, + "tokens_seen": 1364787200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029622868605817457, + "loss": 2.6149, + "theoretical_loss": 3.5457541763374376, + "tokens_seen": 1364787200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962186559679037, + "loss": 2.8065, + "theoretical_loss": 3.5457388913110033, + "tokens_seen": 1364852736 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029620862587763293, + "loss": 2.678, + "theoretical_loss": 3.545723607223983, + "tokens_seen": 1364918272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029619859578736206, + "loss": 2.706, + "theoretical_loss": 3.5457083240762737, + "tokens_seen": 1364983808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961885656970913, + "loss": 2.7492, + "theoretical_loss": 3.545693041867773, + "tokens_seen": 1365049344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961785356068205, + "loss": 2.7696, + "theoretical_loss": 3.545677760598377, + "tokens_seen": 1365114880 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029616850551654966, + "loss": 2.4822, + "theoretical_loss": 3.5456624802679837, + "tokens_seen": 1365180416 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029615847542627884, + "loss": 2.8991, + "theoretical_loss": 3.5456472008764903, + "tokens_seen": 1365245952 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296148445336008, + "loss": 2.6352, + "theoretical_loss": 3.5456319224237944, + "tokens_seen": 1365311488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961384152457372, + "loss": 2.4082, + "theoretical_loss": 3.545616644909792, + "tokens_seen": 1365377024 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029612838515546643, + "loss": 2.7084, + "theoretical_loss": 3.545601368334382, + "tokens_seen": 1365442560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029611835506519556, + "loss": 2.687, + "theoretical_loss": 3.5455860926974605, + "tokens_seen": 1365508096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961083249749248, + "loss": 2.5229, + "theoretical_loss": 3.545570817998925, + "tokens_seen": 1365573632 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296098294884654, + "loss": 2.9669, + "theoretical_loss": 3.5455555442386735, + "tokens_seen": 1365639168 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029608826479438316, + "loss": 2.5514, + "theoretical_loss": 3.5455402714166024, + "tokens_seen": 1365704704 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029607823470411234, + "loss": 2.6351, + "theoretical_loss": 3.54552499953261, + "tokens_seen": 1365770240 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960682046138415, + "loss": 2.584, + "theoretical_loss": 3.545509728586593, + "tokens_seen": 1365835776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960581745235707, + "loss": 2.574, + "theoretical_loss": 3.5454944585784487, + "tokens_seen": 1365901312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029604814443329994, + "loss": 2.6679, + "theoretical_loss": 3.545479189508075, + "tokens_seen": 1365966848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029603811434302906, + "loss": 2.5933, + "theoretical_loss": 3.5454639213753687, + "tokens_seen": 1366032384 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960280842527583, + "loss": 2.781, + "theoretical_loss": 3.545448654180228, + "tokens_seen": 1366097920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960180541624874, + "loss": 2.5818, + "theoretical_loss": 3.54543338792255, + "tokens_seen": 1366163456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029600802407221666, + "loss": 2.5303, + "theoretical_loss": 3.545418122602232, + "tokens_seen": 1366228992 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029599799398194584, + "loss": 2.5922, + "theoretical_loss": 3.5454028582191723, + "tokens_seen": 1366294528 + }, + { + "epoch": 0.41, + "learning_rate": 0.000295987963891675, + "loss": 2.7615, + "theoretical_loss": 3.545387594773267, + "tokens_seen": 1366360064 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 752972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.119584321975708, + "objective/train/theoretical_loss": 3.5453723322644146, + "objective/train/tokens_used": 1386885600, + "theoretical_loss": 3.5453723322644146, + "tokens_seen": 1366425600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959779338014042, + "loss": 2.7982, + "theoretical_loss": 3.5453723322644146, + "tokens_seen": 1366425600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959679037111334, + "loss": 2.4731, + "theoretical_loss": 3.5453570706925124, + "tokens_seen": 1366491136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029595787362086257, + "loss": 2.6281, + "theoretical_loss": 3.545341810057458, + "tokens_seen": 1366556672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959478435305918, + "loss": 2.5008, + "theoretical_loss": 3.5453265503591487, + "tokens_seen": 1366622208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029593781344032093, + "loss": 2.5527, + "theoretical_loss": 3.545311291597483, + "tokens_seen": 1366687744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029592778335005016, + "loss": 2.6509, + "theoretical_loss": 3.545296033772357, + "tokens_seen": 1366753280 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029591775325977935, + "loss": 2.6369, + "theoretical_loss": 3.5452807768836694, + "tokens_seen": 1366818816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029590772316950853, + "loss": 2.7679, + "theoretical_loss": 3.5452655209313173, + "tokens_seen": 1366884352 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958976930792377, + "loss": 2.6728, + "theoretical_loss": 3.545250265915199, + "tokens_seen": 1366949888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958876629889669, + "loss": 2.7048, + "theoretical_loss": 3.5452350118352114, + "tokens_seen": 1367015424 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029587763289869607, + "loss": 2.7995, + "theoretical_loss": 3.5452197586912524, + "tokens_seen": 1367080960 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958676028084253, + "loss": 2.5238, + "theoretical_loss": 3.5452045064832207, + "tokens_seen": 1367146496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029585757271815443, + "loss": 2.6935, + "theoretical_loss": 3.545189255211012, + "tokens_seen": 1367212032 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029584754262788367, + "loss": 2.6453, + "theoretical_loss": 3.545174004874526, + "tokens_seen": 1367277568 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958375125376128, + "loss": 2.6736, + "theoretical_loss": 3.5451587554736594, + "tokens_seen": 1367343104 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029582748244734203, + "loss": 2.7711, + "theoretical_loss": 3.5451435070083104, + "tokens_seen": 1367408640 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958174523570712, + "loss": 2.6743, + "theoretical_loss": 3.545128259478376, + "tokens_seen": 1367474176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958074222668004, + "loss": 2.7954, + "theoretical_loss": 3.5451130128837547, + "tokens_seen": 1367539712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957973921765296, + "loss": 2.6541, + "theoretical_loss": 3.5450977672243447, + "tokens_seen": 1367605248 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957873620862588, + "loss": 2.7035, + "theoretical_loss": 3.5450825225000426, + "tokens_seen": 1367670784 + }, + { + "epoch": 0.41, + "learning_rate": 0.000295777331995988, + "loss": 2.8651, + "theoretical_loss": 3.545067278710747, + "tokens_seen": 1367736320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029576730190571717, + "loss": 2.5355, + "theoretical_loss": 3.5450520358563558, + "tokens_seen": 1367801856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029575727181544635, + "loss": 2.5466, + "theoretical_loss": 3.5450367939367666, + "tokens_seen": 1367867392 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029574724172517553, + "loss": 2.6883, + "theoretical_loss": 3.5450215529518774, + "tokens_seen": 1367932928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029573721163490477, + "loss": 2.4825, + "theoretical_loss": 3.5450063129015863, + "tokens_seen": 1367998464 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 754199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.419557809829712, + "objective/train/theoretical_loss": 3.5449910737857913, + "objective/train/tokens_used": 1388524000, + "theoretical_loss": 3.5449910737857913, + "tokens_seen": 1368064000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957271815446339, + "loss": 2.7239, + "theoretical_loss": 3.5449910737857913, + "tokens_seen": 1368064000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029571715145436313, + "loss": 2.6982, + "theoretical_loss": 3.5449758356043897, + "tokens_seen": 1368129536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029570712136409226, + "loss": 2.6743, + "theoretical_loss": 3.54496059835728, + "tokens_seen": 1368195072 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956970912738215, + "loss": 2.7385, + "theoretical_loss": 3.54494536204436, + "tokens_seen": 1368260608 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956870611835507, + "loss": 2.6535, + "theoretical_loss": 3.5449301266655273, + "tokens_seen": 1368326144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029567703109327986, + "loss": 2.6099, + "theoretical_loss": 3.544914892220681, + "tokens_seen": 1368391680 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029566700100300904, + "loss": 2.962, + "theoretical_loss": 3.544899658709718, + "tokens_seen": 1368457216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956569709127382, + "loss": 2.5697, + "theoretical_loss": 3.5448844261325365, + "tokens_seen": 1368522752 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956469408224674, + "loss": 2.849, + "theoretical_loss": 3.544869194489035, + "tokens_seen": 1368588288 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029563691073219663, + "loss": 2.6937, + "theoretical_loss": 3.5448539637791114, + "tokens_seen": 1368653824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029562688064192576, + "loss": 2.9566, + "theoretical_loss": 3.5448387340026635, + "tokens_seen": 1368719360 + }, + { + "epoch": 0.41, + "learning_rate": 0.000295616850551655, + "loss": 2.5097, + "theoretical_loss": 3.5448235051595898, + "tokens_seen": 1368784896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956068204613842, + "loss": 2.6633, + "theoretical_loss": 3.5448082772497886, + "tokens_seen": 1368850432 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029559679037111336, + "loss": 3.0324, + "theoretical_loss": 3.5447930502731575, + "tokens_seen": 1368915968 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029558676028084254, + "loss": 2.7005, + "theoretical_loss": 3.544777824229594, + "tokens_seen": 1368981504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955767301905717, + "loss": 2.6599, + "theoretical_loss": 3.544762599118998, + "tokens_seen": 1369047040 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955667001003009, + "loss": 2.5976, + "theoretical_loss": 3.5447473749412666, + "tokens_seen": 1369112576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029555667001003014, + "loss": 2.7322, + "theoretical_loss": 3.544732151696298, + "tokens_seen": 1369178112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029554663991975926, + "loss": 2.6484, + "theoretical_loss": 3.5447169293839904, + "tokens_seen": 1369243648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955366098294885, + "loss": 2.7548, + "theoretical_loss": 3.5447017080042422, + "tokens_seen": 1369309184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955265797392176, + "loss": 2.7891, + "theoretical_loss": 3.5446864875569517, + "tokens_seen": 1369374720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029551654964894686, + "loss": 2.6879, + "theoretical_loss": 3.544671268042017, + "tokens_seen": 1369440256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029550651955867604, + "loss": 2.529, + "theoretical_loss": 3.544656049459337, + "tokens_seen": 1369505792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954964894684052, + "loss": 2.8846, + "theoretical_loss": 3.5446408318088087, + "tokens_seen": 1369571328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954864593781344, + "loss": 2.5813, + "theoretical_loss": 3.5446256150903315, + "tokens_seen": 1369636864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 755490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9412763118743896, + "objective/train/theoretical_loss": 3.544610399303803, + "objective/train/tokens_used": 1390162400, + "theoretical_loss": 3.544610399303803, + "tokens_seen": 1369702400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954764292878636, + "loss": 2.6904, + "theoretical_loss": 3.544610399303803, + "tokens_seen": 1369702400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029546639919759277, + "loss": 2.6165, + "theoretical_loss": 3.544595184449122, + "tokens_seen": 1369767936 + }, + { + "epoch": 0.42, + "learning_rate": 0.000295456369107322, + "loss": 2.7068, + "theoretical_loss": 3.544579970526187, + "tokens_seen": 1369833472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029544633901705113, + "loss": 2.4895, + "theoretical_loss": 3.544564757534896, + "tokens_seen": 1369899008 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029543630892678037, + "loss": 2.6439, + "theoretical_loss": 3.5445495454751477, + "tokens_seen": 1369964544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029542627883650955, + "loss": 2.5897, + "theoretical_loss": 3.5445343343468396, + "tokens_seen": 1370030080 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029541624874623873, + "loss": 2.5313, + "theoretical_loss": 3.5445191241498715, + "tokens_seen": 1370095616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954062186559679, + "loss": 2.6542, + "theoretical_loss": 3.5445039148841406, + "tokens_seen": 1370161152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953961885656971, + "loss": 2.6015, + "theoretical_loss": 3.5444887065495463, + "tokens_seen": 1370226688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029538615847542627, + "loss": 2.9512, + "theoretical_loss": 3.5444734991459868, + "tokens_seen": 1370292224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953761283851555, + "loss": 2.8088, + "theoretical_loss": 3.54445829267336, + "tokens_seen": 1370357760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029536609829488463, + "loss": 2.7238, + "theoretical_loss": 3.544443087131565, + "tokens_seen": 1370423296 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029535606820461387, + "loss": 2.5758, + "theoretical_loss": 3.5444278825205, + "tokens_seen": 1370488832 + }, + { + "epoch": 0.42, + "learning_rate": 0.000295346038114343, + "loss": 2.8293, + "theoretical_loss": 3.5444126788400636, + "tokens_seen": 1370554368 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029533600802407223, + "loss": 2.5181, + "theoretical_loss": 3.544397476090155, + "tokens_seen": 1370619904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953259779338014, + "loss": 2.5581, + "theoretical_loss": 3.5443822742706717, + "tokens_seen": 1370685440 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953159478435306, + "loss": 2.5342, + "theoretical_loss": 3.5443670733815127, + "tokens_seen": 1370750976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953059177532598, + "loss": 2.7378, + "theoretical_loss": 3.5443518734225767, + "tokens_seen": 1370816512 + }, + { + "epoch": 0.42, + "learning_rate": 0.000295295887662989, + "loss": 2.709, + "theoretical_loss": 3.544336674393762, + "tokens_seen": 1370882048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029528585757271814, + "loss": 2.4601, + "theoretical_loss": 3.544321476294968, + "tokens_seen": 1370947584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029527582748244737, + "loss": 2.5891, + "theoretical_loss": 3.544306279126092, + "tokens_seen": 1371013120 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952657973921765, + "loss": 2.6917, + "theoretical_loss": 3.5442910828870344, + "tokens_seen": 1371078656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029525576730190573, + "loss": 2.6312, + "theoretical_loss": 3.544275887577692, + "tokens_seen": 1371144192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952457372116349, + "loss": 2.6885, + "theoretical_loss": 3.544260693197965, + "tokens_seen": 1371209728 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952357071213641, + "loss": 2.7528, + "theoretical_loss": 3.5442454997477513, + "tokens_seen": 1371275264 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 756189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.601017951965332, + "objective/train/theoretical_loss": 3.5442303072269503, + "objective/train/tokens_used": 1391800800, + "theoretical_loss": 3.5442303072269503, + "tokens_seen": 1371340800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952256770310933, + "loss": 2.5892, + "theoretical_loss": 3.5442303072269503, + "tokens_seen": 1371340800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029521564694082246, + "loss": 2.5543, + "theoretical_loss": 3.5442151156354598, + "tokens_seen": 1371406336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029520561685055164, + "loss": 2.5929, + "theoretical_loss": 3.544199924973179, + "tokens_seen": 1371471872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951955867602809, + "loss": 2.5026, + "theoretical_loss": 3.5441847352400067, + "tokens_seen": 1371537408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029518555667001, + "loss": 2.7645, + "theoretical_loss": 3.5441695464358416, + "tokens_seen": 1371602944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029517552657973924, + "loss": 2.8162, + "theoretical_loss": 3.544154358560583, + "tokens_seen": 1371668480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029516549648946836, + "loss": 2.7797, + "theoretical_loss": 3.5441391716141286, + "tokens_seen": 1371734016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951554663991976, + "loss": 2.6188, + "theoretical_loss": 3.544123985596378, + "tokens_seen": 1371799552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951454363089268, + "loss": 2.4542, + "theoretical_loss": 3.5441088005072303, + "tokens_seen": 1371865088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029513540621865596, + "loss": 2.518, + "theoretical_loss": 3.544093616346584, + "tokens_seen": 1371930624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029512537612838514, + "loss": 2.4409, + "theoretical_loss": 3.544078433114338, + "tokens_seen": 1371996160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951153460381144, + "loss": 2.7041, + "theoretical_loss": 3.5440632508103906, + "tokens_seen": 1372061696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951053159478435, + "loss": 2.5683, + "theoretical_loss": 3.544048069434642, + "tokens_seen": 1372127232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029509528585757274, + "loss": 2.5049, + "theoretical_loss": 3.5440328889869903, + "tokens_seen": 1372192768 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029508525576730187, + "loss": 2.9035, + "theoretical_loss": 3.5440177094673344, + "tokens_seen": 1372258304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950752256770311, + "loss": 2.6459, + "theoretical_loss": 3.544002530875573, + "tokens_seen": 1372323840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950651955867603, + "loss": 2.8768, + "theoretical_loss": 3.543987353211606, + "tokens_seen": 1372389376 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029505516549648946, + "loss": 2.8668, + "theoretical_loss": 3.543972176475332, + "tokens_seen": 1372454912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029504513540621865, + "loss": 2.6032, + "theoretical_loss": 3.543957000666649, + "tokens_seen": 1372520448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950351053159478, + "loss": 2.676, + "theoretical_loss": 3.543941825785458, + "tokens_seen": 1372585984 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029502507522567706, + "loss": 2.6996, + "theoretical_loss": 3.543926651831656, + "tokens_seen": 1372651520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029501504513540624, + "loss": 2.8494, + "theoretical_loss": 3.5439114788051436, + "tokens_seen": 1372717056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950050150451354, + "loss": 2.4998, + "theoretical_loss": 3.543896306705819, + "tokens_seen": 1372782592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949949849548646, + "loss": 2.6772, + "theoretical_loss": 3.543881135533581, + "tokens_seen": 1372848128 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949849548645938, + "loss": 2.8906, + "theoretical_loss": 3.5438659652883295, + "tokens_seen": 1372913664 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 757380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.386281967163086, + "objective/train/theoretical_loss": 3.5438507959699637, + "objective/train/tokens_used": 1393439200, + "theoretical_loss": 3.5438507959699637, + "tokens_seen": 1372979200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029497492477432297, + "loss": 2.5829, + "theoretical_loss": 3.5438507959699637, + "tokens_seen": 1372979200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949648946840522, + "loss": 2.7449, + "theoretical_loss": 3.5438356275783827, + "tokens_seen": 1373044736 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029495486459378133, + "loss": 2.6824, + "theoretical_loss": 3.5438204601134844, + "tokens_seen": 1373110272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029494483450351057, + "loss": 2.6819, + "theoretical_loss": 3.5438052935751694, + "tokens_seen": 1373175808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029493480441323975, + "loss": 2.5504, + "theoretical_loss": 3.5437901279633364, + "tokens_seen": 1373241344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029492477432296893, + "loss": 2.6439, + "theoretical_loss": 3.543774963277884, + "tokens_seen": 1373306880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949147442326981, + "loss": 2.6087, + "theoretical_loss": 3.5437597995187122, + "tokens_seen": 1373372416 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949047141424273, + "loss": 2.3426, + "theoretical_loss": 3.54374463668572, + "tokens_seen": 1373437952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029489468405215647, + "loss": 2.615, + "theoretical_loss": 3.5437294747788064, + "tokens_seen": 1373503488 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948846539618857, + "loss": 2.4632, + "theoretical_loss": 3.5437143137978713, + "tokens_seen": 1373569024 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029487462387161483, + "loss": 2.5999, + "theoretical_loss": 3.5436991537428133, + "tokens_seen": 1373634560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029486459378134407, + "loss": 2.5646, + "theoretical_loss": 3.5436839946135317, + "tokens_seen": 1373700096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948545636910732, + "loss": 2.4645, + "theoretical_loss": 3.5436688364099265, + "tokens_seen": 1373765632 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029484453360080243, + "loss": 2.7137, + "theoretical_loss": 3.5436536791318964, + "tokens_seen": 1373831168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948345035105316, + "loss": 2.779, + "theoretical_loss": 3.5436385227793408, + "tokens_seen": 1373896704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948244734202608, + "loss": 2.5723, + "theoretical_loss": 3.543623367352159, + "tokens_seen": 1373962240 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029481444332999, + "loss": 2.4863, + "theoretical_loss": 3.5436082128502506, + "tokens_seen": 1374027776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948044132397192, + "loss": 2.4921, + "theoretical_loss": 3.543593059273515, + "tokens_seen": 1374093312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029479438314944834, + "loss": 2.5952, + "theoretical_loss": 3.5435779066218513, + "tokens_seen": 1374158848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029478435305917757, + "loss": 2.6188, + "theoretical_loss": 3.5435627548951594, + "tokens_seen": 1374224384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947743229689067, + "loss": 2.6058, + "theoretical_loss": 3.543547604093338, + "tokens_seen": 1374289920 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029476429287863593, + "loss": 2.7356, + "theoretical_loss": 3.543532454216287, + "tokens_seen": 1374355456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947542627883651, + "loss": 2.7198, + "theoretical_loss": 3.543517305263906, + "tokens_seen": 1374420992 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947442326980943, + "loss": 2.7524, + "theoretical_loss": 3.5435021572360945, + "tokens_seen": 1374486528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947342026078235, + "loss": 2.8701, + "theoretical_loss": 3.5434870101327514, + "tokens_seen": 1374552064 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 758052, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.612107992172241, + "objective/train/theoretical_loss": 3.5434718639537763, + "objective/train/tokens_used": 1395077600, + "theoretical_loss": 3.5434718639537763, + "tokens_seen": 1374617600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029472417251755266, + "loss": 2.6773, + "theoretical_loss": 3.5434718639537763, + "tokens_seen": 1374617600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029471414242728184, + "loss": 2.6392, + "theoretical_loss": 3.54345671869907, + "tokens_seen": 1374683136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947041123370111, + "loss": 2.4857, + "theoretical_loss": 3.54344157436853, + "tokens_seen": 1374748672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946940822467402, + "loss": 2.5248, + "theoretical_loss": 3.5434264309620573, + "tokens_seen": 1374814208 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029468405215646944, + "loss": 2.6708, + "theoretical_loss": 3.5434112884795517, + "tokens_seen": 1374879744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029467402206619856, + "loss": 2.6134, + "theoretical_loss": 3.5433961469209114, + "tokens_seen": 1374945280 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946639919759278, + "loss": 2.6086, + "theoretical_loss": 3.543381006286037, + "tokens_seen": 1375010816 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294653961885657, + "loss": 2.6643, + "theoretical_loss": 3.5433658665748284, + "tokens_seen": 1375076352 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029464393179538616, + "loss": 2.8531, + "theoretical_loss": 3.543350727787184, + "tokens_seen": 1375141888 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029463390170511534, + "loss": 2.8589, + "theoretical_loss": 3.5433355899230046, + "tokens_seen": 1375207424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946238716148446, + "loss": 2.6566, + "theoretical_loss": 3.543320452982189, + "tokens_seen": 1375272960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946138415245737, + "loss": 2.6606, + "theoretical_loss": 3.543305316964638, + "tokens_seen": 1375338496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029460381143430294, + "loss": 2.6512, + "theoretical_loss": 3.5432901818702502, + "tokens_seen": 1375404032 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029459378134403207, + "loss": 2.7956, + "theoretical_loss": 3.5432750476989257, + "tokens_seen": 1375469568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945837512537613, + "loss": 2.6258, + "theoretical_loss": 3.543259914450564, + "tokens_seen": 1375535104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945737211634905, + "loss": 2.6525, + "theoretical_loss": 3.5432447821250657, + "tokens_seen": 1375600640 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029456369107321966, + "loss": 2.5247, + "theoretical_loss": 3.5432296507223295, + "tokens_seen": 1375666176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029455366098294885, + "loss": 2.5934, + "theoretical_loss": 3.5432145202422562, + "tokens_seen": 1375731712 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294543630892678, + "loss": 2.6191, + "theoretical_loss": 3.5431993906847445, + "tokens_seen": 1375797248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945336008024072, + "loss": 2.7456, + "theoretical_loss": 3.543184262049695, + "tokens_seen": 1375862784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029452357071213644, + "loss": 2.5578, + "theoretical_loss": 3.543169134337007, + "tokens_seen": 1375928320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029451354062186557, + "loss": 2.9169, + "theoretical_loss": 3.5431540075465806, + "tokens_seen": 1375993856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945035105315948, + "loss": 2.7079, + "theoretical_loss": 3.543138881678316, + "tokens_seen": 1376059392 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029449348044132393, + "loss": 2.4352, + "theoretical_loss": 3.543123756732112, + "tokens_seen": 1376124928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029448345035105317, + "loss": 2.5168, + "theoretical_loss": 3.54310863270787, + "tokens_seen": 1376190464 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 759249, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.637690305709839, + "objective/train/theoretical_loss": 3.5430935096054883, + "objective/train/tokens_used": 1396716000, + "theoretical_loss": 3.5430935096054883, + "tokens_seen": 1376256000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029447342026078235, + "loss": 2.5261, + "theoretical_loss": 3.5430935096054883, + "tokens_seen": 1376256000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029446339017051153, + "loss": 2.5534, + "theoretical_loss": 3.5430783874248686, + "tokens_seen": 1376321536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944533600802407, + "loss": 2.5707, + "theoretical_loss": 3.543063266165909, + "tokens_seen": 1376387072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029444332998996995, + "loss": 2.5748, + "theoretical_loss": 3.543048145828511, + "tokens_seen": 1376452608 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944332998996991, + "loss": 2.6966, + "theoretical_loss": 3.543033026412573, + "tokens_seen": 1376518144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944232698094283, + "loss": 2.5546, + "theoretical_loss": 3.5430179079179958, + "tokens_seen": 1376583680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029441323971915744, + "loss": 2.5579, + "theoretical_loss": 3.54300279034468, + "tokens_seen": 1376649216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029440320962888667, + "loss": 2.4806, + "theoretical_loss": 3.542987673692525, + "tokens_seen": 1376714752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029439317953861585, + "loss": 2.6588, + "theoretical_loss": 3.5429725579614306, + "tokens_seen": 1376780288 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029438314944834503, + "loss": 2.7941, + "theoretical_loss": 3.5429574431512973, + "tokens_seen": 1376845824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943731193580742, + "loss": 2.6557, + "theoretical_loss": 3.542942329262025, + "tokens_seen": 1376911360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943630892678034, + "loss": 2.6905, + "theoretical_loss": 3.5429272162935135, + "tokens_seen": 1376976896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943530591775326, + "loss": 2.5786, + "theoretical_loss": 3.542912104245663, + "tokens_seen": 1377042432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943430290872618, + "loss": 2.6899, + "theoretical_loss": 3.542896993118374, + "tokens_seen": 1377107968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029433299899699094, + "loss": 2.6485, + "theoretical_loss": 3.542881882911546, + "tokens_seen": 1377173504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943229689067202, + "loss": 2.6091, + "theoretical_loss": 3.5428667736250796, + "tokens_seen": 1377239040 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943129388164493, + "loss": 2.5071, + "theoretical_loss": 3.5428516652588753, + "tokens_seen": 1377304576 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029430290872617854, + "loss": 2.6595, + "theoretical_loss": 3.542836557812832, + "tokens_seen": 1377370112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029429287863590777, + "loss": 2.6319, + "theoretical_loss": 3.542821451286851, + "tokens_seen": 1377435648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942828485456369, + "loss": 2.3688, + "theoretical_loss": 3.542806345680832, + "tokens_seen": 1377501184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029427281845536613, + "loss": 2.643, + "theoretical_loss": 3.5427912409946756, + "tokens_seen": 1377566720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942627883650953, + "loss": 2.614, + "theoretical_loss": 3.542776137228282, + "tokens_seen": 1377632256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942527582748245, + "loss": 2.5967, + "theoretical_loss": 3.5427610343815505, + "tokens_seen": 1377697792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942427281845537, + "loss": 2.6822, + "theoretical_loss": 3.542745932454383, + "tokens_seen": 1377763328 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029423269809428286, + "loss": 2.5199, + "theoretical_loss": 3.5427308314466783, + "tokens_seen": 1377828864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 760358, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.197251319885254, + "objective/train/theoretical_loss": 3.542715731358337, + "objective/train/tokens_used": 1398354400, + "theoretical_loss": 3.542715731358337, + "tokens_seen": 1377894400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029422266800401204, + "loss": 2.4138, + "theoretical_loss": 3.542715731358337, + "tokens_seen": 1377894400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942126379137413, + "loss": 2.4419, + "theoretical_loss": 3.54270063218926, + "tokens_seen": 1377959936 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942026078234704, + "loss": 2.6702, + "theoretical_loss": 3.5426855339393475, + "tokens_seen": 1378025472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029419257773319964, + "loss": 2.6982, + "theoretical_loss": 3.5426704366084993, + "tokens_seen": 1378091008 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029418254764292876, + "loss": 2.4193, + "theoretical_loss": 3.542655340196616, + "tokens_seen": 1378156544 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294172517552658, + "loss": 2.7807, + "theoretical_loss": 3.5426402447035983, + "tokens_seen": 1378222080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941624874623872, + "loss": 2.7231, + "theoretical_loss": 3.5426251501293464, + "tokens_seen": 1378287616 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029415245737211636, + "loss": 2.6748, + "theoretical_loss": 3.5426100564737606, + "tokens_seen": 1378353152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029414242728184554, + "loss": 2.716, + "theoretical_loss": 3.542594963736741, + "tokens_seen": 1378418688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941323971915748, + "loss": 2.754, + "theoretical_loss": 3.5425798719181882, + "tokens_seen": 1378484224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941223671013039, + "loss": 2.5394, + "theoretical_loss": 3.5425647810180037, + "tokens_seen": 1378549760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029411233701103314, + "loss": 2.621, + "theoretical_loss": 3.5425496910360863, + "tokens_seen": 1378615296 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029410230692076227, + "loss": 2.585, + "theoretical_loss": 3.5425346019723376, + "tokens_seen": 1378680832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940922768304915, + "loss": 2.5968, + "theoretical_loss": 3.542519513826657, + "tokens_seen": 1378746368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940822467402207, + "loss": 2.478, + "theoretical_loss": 3.5425044265989465, + "tokens_seen": 1378811904 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029407221664994986, + "loss": 2.7627, + "theoretical_loss": 3.542489340289106, + "tokens_seen": 1378877440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029406218655967905, + "loss": 2.4349, + "theoretical_loss": 3.5424742548970354, + "tokens_seen": 1378942976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940521564694082, + "loss": 2.6054, + "theoretical_loss": 3.542459170422636, + "tokens_seen": 1379008512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940421263791374, + "loss": 2.7946, + "theoretical_loss": 3.542444086865808, + "tokens_seen": 1379074048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029403209628886664, + "loss": 2.7524, + "theoretical_loss": 3.5424290042264523, + "tokens_seen": 1379139584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029402206619859577, + "loss": 2.8018, + "theoretical_loss": 3.542413922504469, + "tokens_seen": 1379205120 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294012036108325, + "loss": 2.7234, + "theoretical_loss": 3.542398841699759, + "tokens_seen": 1379270656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029400200601805413, + "loss": 2.4455, + "theoretical_loss": 3.5423837618122236, + "tokens_seen": 1379336192 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029399197592778337, + "loss": 2.5743, + "theoretical_loss": 3.5423686828417624, + "tokens_seen": 1379401728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029398194583751255, + "loss": 2.6166, + "theoretical_loss": 3.542353604788276, + "tokens_seen": 1379467264 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 760756, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8100745677948, + "objective/train/theoretical_loss": 3.5423385276516663, + "objective/train/tokens_used": 1399992800, + "theoretical_loss": 3.5423385276516663, + "tokens_seen": 1379532800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029397191574724173, + "loss": 2.5965, + "theoretical_loss": 3.5423385276516663, + "tokens_seen": 1379532800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939618856569709, + "loss": 2.5801, + "theoretical_loss": 3.542323451431833, + "tokens_seen": 1379598336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029395185556670015, + "loss": 2.5508, + "theoretical_loss": 3.542308376128677, + "tokens_seen": 1379663872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939418254764293, + "loss": 2.4807, + "theoretical_loss": 3.5422933017420988, + "tokens_seen": 1379729408 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939317953861585, + "loss": 2.7529, + "theoretical_loss": 3.542278228271999, + "tokens_seen": 1379794944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029392176529588764, + "loss": 2.5369, + "theoretical_loss": 3.5422631557182798, + "tokens_seen": 1379860480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029391173520561687, + "loss": 2.6885, + "theoretical_loss": 3.542248084080841, + "tokens_seen": 1379926016 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029390170511534605, + "loss": 2.6143, + "theoretical_loss": 3.5422330133595823, + "tokens_seen": 1379991552 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029389167502507523, + "loss": 2.5206, + "theoretical_loss": 3.542217943554406, + "tokens_seen": 1380057088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938816449348044, + "loss": 2.5229, + "theoretical_loss": 3.5422028746652123, + "tokens_seen": 1380122624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938716148445336, + "loss": 2.7405, + "theoretical_loss": 3.542187806691903, + "tokens_seen": 1380188160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938615847542628, + "loss": 2.507, + "theoretical_loss": 3.5421727396343767, + "tokens_seen": 1380253696 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293851554663992, + "loss": 2.6167, + "theoretical_loss": 3.542157673492537, + "tokens_seen": 1380319232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029384152457372114, + "loss": 2.7905, + "theoretical_loss": 3.542142608266283, + "tokens_seen": 1380384768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938314944834504, + "loss": 2.7074, + "theoretical_loss": 3.542127543955515, + "tokens_seen": 1380450304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938214643931795, + "loss": 2.5172, + "theoretical_loss": 3.542112480560136, + "tokens_seen": 1380515840 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029381143430290874, + "loss": 2.6529, + "theoretical_loss": 3.542097418080046, + "tokens_seen": 1380581376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938014042126379, + "loss": 2.8394, + "theoretical_loss": 3.5420823565151456, + "tokens_seen": 1380646912 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937913741223671, + "loss": 2.6438, + "theoretical_loss": 3.542067295865336, + "tokens_seen": 1380712448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937813440320963, + "loss": 2.588, + "theoretical_loss": 3.542052236130518, + "tokens_seen": 1380777984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937713139418255, + "loss": 2.888, + "theoretical_loss": 3.542037177310593, + "tokens_seen": 1380843520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029376128385155464, + "loss": 2.8485, + "theoretical_loss": 3.5420221194054617, + "tokens_seen": 1380909056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937512537612839, + "loss": 2.633, + "theoretical_loss": 3.542007062415025, + "tokens_seen": 1380974592 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293741223671013, + "loss": 2.4821, + "theoretical_loss": 3.541992006339184, + "tokens_seen": 1381040128 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029373119358074224, + "loss": 2.6478, + "theoretical_loss": 3.54197695117784, + "tokens_seen": 1381105664 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 761793, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.823936700820923, + "objective/train/theoretical_loss": 3.5419618969308937, + "objective/train/tokens_used": 1401631200, + "theoretical_loss": 3.5419618969308937, + "tokens_seen": 1381171200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937211634904714, + "loss": 2.7591, + "theoretical_loss": 3.5419618969308937, + "tokens_seen": 1381171200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937111334002006, + "loss": 2.7321, + "theoretical_loss": 3.5419468435982466, + "tokens_seen": 1381236736 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937011033099298, + "loss": 2.7409, + "theoretical_loss": 3.5419317911798, + "tokens_seen": 1381302272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029369107321965896, + "loss": 2.6413, + "theoretical_loss": 3.541916739675454, + "tokens_seen": 1381367808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029368104312938815, + "loss": 2.6433, + "theoretical_loss": 3.54190168908511, + "tokens_seen": 1381433344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936710130391174, + "loss": 2.8848, + "theoretical_loss": 3.54188663940867, + "tokens_seen": 1381498880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936609829488465, + "loss": 2.677, + "theoretical_loss": 3.5418715906460343, + "tokens_seen": 1381564416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029365095285857574, + "loss": 2.5477, + "theoretical_loss": 3.5418565427971043, + "tokens_seen": 1381629952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029364092276830487, + "loss": 2.5925, + "theoretical_loss": 3.5418414958617817, + "tokens_seen": 1381695488 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936308926780341, + "loss": 2.6368, + "theoretical_loss": 3.541826449839967, + "tokens_seen": 1381761024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936208625877633, + "loss": 2.4387, + "theoretical_loss": 3.5418114047315616, + "tokens_seen": 1381826560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029361083249749247, + "loss": 2.7707, + "theoretical_loss": 3.541796360536467, + "tokens_seen": 1381892096 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029360080240722165, + "loss": 2.6518, + "theoretical_loss": 3.5417813172545842, + "tokens_seen": 1381957632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935907723169509, + "loss": 2.5689, + "theoretical_loss": 3.5417662748858145, + "tokens_seen": 1382023168 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029358074222668, + "loss": 2.5661, + "theoretical_loss": 3.541751233430059, + "tokens_seen": 1382088704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029357071213640925, + "loss": 2.5498, + "theoretical_loss": 3.5417361928872193, + "tokens_seen": 1382154240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935606820461384, + "loss": 2.6417, + "theoretical_loss": 3.541721153257197, + "tokens_seen": 1382219776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935506519558676, + "loss": 2.6782, + "theoretical_loss": 3.5417061145398927, + "tokens_seen": 1382285312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029354062186559684, + "loss": 2.8397, + "theoretical_loss": 3.541691076735208, + "tokens_seen": 1382350848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029353059177532597, + "loss": 2.495, + "theoretical_loss": 3.5416760398430442, + "tokens_seen": 1382416384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935205616850552, + "loss": 2.8524, + "theoretical_loss": 3.5416610038633034, + "tokens_seen": 1382481920 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029351053159478433, + "loss": 2.7181, + "theoretical_loss": 3.541645968795886, + "tokens_seen": 1382547456 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029350050150451357, + "loss": 2.7833, + "theoretical_loss": 3.541630934640694, + "tokens_seen": 1382612992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029349047141424275, + "loss": 2.5108, + "theoretical_loss": 3.541615901397628, + "tokens_seen": 1382678528 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029348044132397193, + "loss": 2.6661, + "theoretical_loss": 3.5416008690665906, + "tokens_seen": 1382744064 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 762358, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7813034057617188, + "objective/train/theoretical_loss": 3.5415858376474825, + "objective/train/tokens_used": 1403269600, + "theoretical_loss": 3.5415858376474825, + "tokens_seen": 1382809600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934704112337011, + "loss": 2.7684, + "theoretical_loss": 3.5415858376474825, + "tokens_seen": 1382809600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029346038114343035, + "loss": 2.6129, + "theoretical_loss": 3.5415708071402054, + "tokens_seen": 1382875136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934503510531595, + "loss": 2.6347, + "theoretical_loss": 3.5415557775446604, + "tokens_seen": 1382940672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934403209628887, + "loss": 2.5538, + "theoretical_loss": 3.5415407488607498, + "tokens_seen": 1383006208 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029343029087261784, + "loss": 2.7402, + "theoretical_loss": 3.5415257210883744, + "tokens_seen": 1383071744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029342026078234707, + "loss": 2.5256, + "theoretical_loss": 3.541510694227436, + "tokens_seen": 1383137280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029341023069207625, + "loss": 2.6425, + "theoretical_loss": 3.541495668277836, + "tokens_seen": 1383202816 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029340020060180543, + "loss": 2.5158, + "theoretical_loss": 3.541480643239476, + "tokens_seen": 1383268352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933901705115346, + "loss": 2.6083, + "theoretical_loss": 3.541465619112258, + "tokens_seen": 1383333888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933801404212638, + "loss": 2.6756, + "theoretical_loss": 3.541450595896083, + "tokens_seen": 1383399424 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293370110330993, + "loss": 2.635, + "theoretical_loss": 3.541435573590852, + "tokens_seen": 1383464960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933600802407222, + "loss": 2.581, + "theoretical_loss": 3.5414205521964686, + "tokens_seen": 1383530496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029335005015045134, + "loss": 2.7649, + "theoretical_loss": 3.5414055317128326, + "tokens_seen": 1383596032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933400200601806, + "loss": 2.5137, + "theoretical_loss": 3.5413905121398463, + "tokens_seen": 1383661568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933299899699097, + "loss": 2.7317, + "theoretical_loss": 3.5413754934774113, + "tokens_seen": 1383727104 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029331995987963894, + "loss": 2.8341, + "theoretical_loss": 3.54136047572543, + "tokens_seen": 1383792640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933099297893681, + "loss": 2.8915, + "theoretical_loss": 3.5413454588838027, + "tokens_seen": 1383858176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932998996990973, + "loss": 2.8376, + "theoretical_loss": 3.541330442952432, + "tokens_seen": 1383923712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932898696088265, + "loss": 2.5754, + "theoretical_loss": 3.5413154279312193, + "tokens_seen": 1383989248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932798395185557, + "loss": 2.5504, + "theoretical_loss": 3.5413004138200668, + "tokens_seen": 1384054784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029326980942828484, + "loss": 2.8385, + "theoretical_loss": 3.541285400618876, + "tokens_seen": 1384120320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932597793380141, + "loss": 2.7744, + "theoretical_loss": 3.5412703883275487, + "tokens_seen": 1384185856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932497492477432, + "loss": 2.6171, + "theoretical_loss": 3.541255376945986, + "tokens_seen": 1384251392 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029323971915747244, + "loss": 2.763, + "theoretical_loss": 3.5412403664740904, + "tokens_seen": 1384316928 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932296890672016, + "loss": 2.5659, + "theoretical_loss": 3.541225356911764, + "tokens_seen": 1384382464 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 763291, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.771702289581299, + "objective/train/theoretical_loss": 3.5412103482589083, + "objective/train/tokens_used": 1404908000, + "theoretical_loss": 3.5412103482589083, + "tokens_seen": 1384448000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932196589769308, + "loss": 2.7747, + "theoretical_loss": 3.5412103482589083, + "tokens_seen": 1384448000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029320962888666, + "loss": 2.5403, + "theoretical_loss": 3.541195340515425, + "tokens_seen": 1384513536 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029319959879638916, + "loss": 2.7576, + "theoretical_loss": 3.5411803336812158, + "tokens_seen": 1384579072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029318956870611835, + "loss": 2.7168, + "theoretical_loss": 3.5411653277561825, + "tokens_seen": 1384644608 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931795386158476, + "loss": 2.7653, + "theoretical_loss": 3.541150322740228, + "tokens_seen": 1384710144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931695085255767, + "loss": 2.7363, + "theoretical_loss": 3.541135318633253, + "tokens_seen": 1384775680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029315947843530594, + "loss": 2.5727, + "theoretical_loss": 3.5411203154351605, + "tokens_seen": 1384841216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029314944834503507, + "loss": 2.5424, + "theoretical_loss": 3.5411053131458514, + "tokens_seen": 1384906752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931394182547643, + "loss": 2.6556, + "theoretical_loss": 3.5410903117652284, + "tokens_seen": 1384972288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931293881644935, + "loss": 2.5225, + "theoretical_loss": 3.5410753112931928, + "tokens_seen": 1385037824 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029311935807422267, + "loss": 2.6571, + "theoretical_loss": 3.5410603117296473, + "tokens_seen": 1385103360 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029310932798395185, + "loss": 2.5987, + "theoretical_loss": 3.5410453130744934, + "tokens_seen": 1385168896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930992978936811, + "loss": 2.544, + "theoretical_loss": 3.5410303153276335, + "tokens_seen": 1385234432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930892678034102, + "loss": 2.7685, + "theoretical_loss": 3.541015318488969, + "tokens_seen": 1385299968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029307923771313945, + "loss": 2.626, + "theoretical_loss": 3.5410003225584026, + "tokens_seen": 1385365504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930692076228686, + "loss": 2.6125, + "theoretical_loss": 3.5409853275358363, + "tokens_seen": 1385431040 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930591775325978, + "loss": 2.4994, + "theoretical_loss": 3.5409703334211717, + "tokens_seen": 1385496576 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293049147442327, + "loss": 2.7026, + "theoretical_loss": 3.5409553402143112, + "tokens_seen": 1385562112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029303911735205617, + "loss": 2.5906, + "theoretical_loss": 3.540940347915157, + "tokens_seen": 1385627648 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029302908726178535, + "loss": 2.7143, + "theoretical_loss": 3.540925356523611, + "tokens_seen": 1385693184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029301905717151453, + "loss": 2.6731, + "theoretical_loss": 3.5409103660395758, + "tokens_seen": 1385758720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930090270812437, + "loss": 2.6631, + "theoretical_loss": 3.5408953764629527, + "tokens_seen": 1385824256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029299899699097295, + "loss": 2.8875, + "theoretical_loss": 3.5408803877936443, + "tokens_seen": 1385889792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929889669007021, + "loss": 2.5833, + "theoretical_loss": 3.540865400031553, + "tokens_seen": 1385955328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929789368104313, + "loss": 2.5372, + "theoretical_loss": 3.540850413176581, + "tokens_seen": 1386020864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 764545, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.004178762435913, + "objective/train/theoretical_loss": 3.5408354272286298, + "objective/train/tokens_used": 1406546400, + "theoretical_loss": 3.5408354272286298, + "tokens_seen": 1386086400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929689067201605, + "loss": 2.9131, + "theoretical_loss": 3.5408354272286298, + "tokens_seen": 1386086400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929588766298897, + "loss": 2.6061, + "theoretical_loss": 3.540820442187602, + "tokens_seen": 1386151936 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029294884653961885, + "loss": 2.6359, + "theoretical_loss": 3.540805458053401, + "tokens_seen": 1386217472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029293881644934804, + "loss": 2.5335, + "theoretical_loss": 3.5407904748259273, + "tokens_seen": 1386283008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929287863590772, + "loss": 2.7497, + "theoretical_loss": 3.5407754925050843, + "tokens_seen": 1386348544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029291875626880645, + "loss": 2.8104, + "theoretical_loss": 3.540760511090774, + "tokens_seen": 1386414080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929087261785356, + "loss": 2.6975, + "theoretical_loss": 3.540745530582898, + "tokens_seen": 1386479616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928986960882648, + "loss": 2.59, + "theoretical_loss": 3.540730550981359, + "tokens_seen": 1386545152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029288866599799394, + "loss": 2.5904, + "theoretical_loss": 3.54071557228606, + "tokens_seen": 1386610688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928786359077232, + "loss": 2.5001, + "theoretical_loss": 3.540700594496903, + "tokens_seen": 1386676224 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029286860581745236, + "loss": 2.5749, + "theoretical_loss": 3.54068561761379, + "tokens_seen": 1386741760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029285857572718154, + "loss": 2.6409, + "theoretical_loss": 3.5406706416366243, + "tokens_seen": 1386807296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928485456369107, + "loss": 2.6421, + "theoretical_loss": 3.540655666565307, + "tokens_seen": 1386872832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928385155466399, + "loss": 2.4422, + "theoretical_loss": 3.540640692399741, + "tokens_seen": 1386938368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928284854563691, + "loss": 2.6959, + "theoretical_loss": 3.540625719139829, + "tokens_seen": 1387003904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928184553660983, + "loss": 2.5246, + "theoretical_loss": 3.5406107467854735, + "tokens_seen": 1387069440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029280842527582744, + "loss": 2.4474, + "theoretical_loss": 3.540595775336577, + "tokens_seen": 1387134976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927983951855567, + "loss": 2.6563, + "theoretical_loss": 3.5405808047930405, + "tokens_seen": 1387200512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927883650952859, + "loss": 2.7903, + "theoretical_loss": 3.540565835154769, + "tokens_seen": 1387266048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029277833500501504, + "loss": 2.6836, + "theoretical_loss": 3.540550866421663, + "tokens_seen": 1387331584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927683049147443, + "loss": 2.59, + "theoretical_loss": 3.5405358985936255, + "tokens_seen": 1387397120 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927582748244734, + "loss": 2.516, + "theoretical_loss": 3.5405209316705597, + "tokens_seen": 1387462656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029274824473420264, + "loss": 2.8805, + "theoretical_loss": 3.5405059656523674, + "tokens_seen": 1387528192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927382146439318, + "loss": 2.623, + "theoretical_loss": 3.5404910005389514, + "tokens_seen": 1387593728 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292728184553661, + "loss": 2.5908, + "theoretical_loss": 3.5404760363302143, + "tokens_seen": 1387659264 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 765178, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3576533794403076, + "objective/train/theoretical_loss": 3.5404610730260595, + "objective/train/tokens_used": 1408184800, + "theoretical_loss": 3.5404610730260595, + "tokens_seen": 1387724800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927181544633902, + "loss": 2.3922, + "theoretical_loss": 3.5404610730260595, + "tokens_seen": 1387724800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029270812437311936, + "loss": 2.5994, + "theoretical_loss": 3.540446110626388, + "tokens_seen": 1387790336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029269809428284855, + "loss": 2.5658, + "theoretical_loss": 3.540431149131103, + "tokens_seen": 1387855872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926880641925778, + "loss": 2.7686, + "theoretical_loss": 3.5404161885401075, + "tokens_seen": 1387921408 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926780341023069, + "loss": 2.4703, + "theoretical_loss": 3.5404012288533044, + "tokens_seen": 1387986944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029266800401203614, + "loss": 2.8259, + "theoretical_loss": 3.5403862700705955, + "tokens_seen": 1388052480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029265797392176527, + "loss": 2.5285, + "theoretical_loss": 3.5403713121918843, + "tokens_seen": 1388118016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926479438314945, + "loss": 2.5711, + "theoretical_loss": 3.5403563552170727, + "tokens_seen": 1388183552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926379137412237, + "loss": 2.849, + "theoretical_loss": 3.540341399146064, + "tokens_seen": 1388249088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029262788365095287, + "loss": 2.4781, + "theoretical_loss": 3.5403264439787607, + "tokens_seen": 1388314624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029261785356068205, + "loss": 2.6438, + "theoretical_loss": 3.540311489715066, + "tokens_seen": 1388380160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926078234704113, + "loss": 2.6651, + "theoretical_loss": 3.5402965363548815, + "tokens_seen": 1388445696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925977933801404, + "loss": 2.7053, + "theoretical_loss": 3.5402815838981114, + "tokens_seen": 1388511232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029258776328986965, + "loss": 2.6005, + "theoretical_loss": 3.5402666323446574, + "tokens_seen": 1388576768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925777331995988, + "loss": 2.5783, + "theoretical_loss": 3.5402516816944227, + "tokens_seen": 1388642304 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292567703109328, + "loss": 2.5394, + "theoretical_loss": 3.54023673194731, + "tokens_seen": 1388707840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925576730190572, + "loss": 2.6339, + "theoretical_loss": 3.5402217831032226, + "tokens_seen": 1388773376 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029254764292878637, + "loss": 2.4864, + "theoretical_loss": 3.5402068351620626, + "tokens_seen": 1388838912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029253761283851555, + "loss": 2.5444, + "theoretical_loss": 3.5401918881237338, + "tokens_seen": 1388904448 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029252758274824473, + "loss": 2.5277, + "theoretical_loss": 3.540176941988138, + "tokens_seen": 1388969984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925175526579739, + "loss": 2.6024, + "theoretical_loss": 3.5401619967551787, + "tokens_seen": 1389035520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029250752256770315, + "loss": 2.5055, + "theoretical_loss": 3.540147052424759, + "tokens_seen": 1389101056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924974924774323, + "loss": 2.4434, + "theoretical_loss": 3.540132108996781, + "tokens_seen": 1389166592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924874623871615, + "loss": 2.9739, + "theoretical_loss": 3.5401171664711484, + "tokens_seen": 1389232128 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924774322968907, + "loss": 2.6789, + "theoretical_loss": 3.5401022248477636, + "tokens_seen": 1389297664 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 766177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7885921001434326, + "objective/train/theoretical_loss": 3.540087284126531, + "objective/train/tokens_used": 1409823200, + "theoretical_loss": 3.540087284126531, + "tokens_seen": 1389363200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924674022066199, + "loss": 2.7667, + "theoretical_loss": 3.540087284126531, + "tokens_seen": 1389363200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029245737211634906, + "loss": 2.6305, + "theoretical_loss": 3.540072344307351, + "tokens_seen": 1389428736 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029244734202607824, + "loss": 2.6864, + "theoretical_loss": 3.540057405390129, + "tokens_seen": 1389494272 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924373119358074, + "loss": 2.5925, + "theoretical_loss": 3.5400424673747666, + "tokens_seen": 1389559808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029242728184553665, + "loss": 2.6318, + "theoretical_loss": 3.5400275302611677, + "tokens_seen": 1389625344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924172517552658, + "loss": 2.6014, + "theoretical_loss": 3.5400125940492346, + "tokens_seen": 1389690880 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292407221664995, + "loss": 2.8096, + "theoretical_loss": 3.5399976587388706, + "tokens_seen": 1389756416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029239719157472414, + "loss": 2.7303, + "theoretical_loss": 3.5399827243299793, + "tokens_seen": 1389821952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923871614844534, + "loss": 2.6735, + "theoretical_loss": 3.5399677908224625, + "tokens_seen": 1389887488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029237713139418256, + "loss": 2.5112, + "theoretical_loss": 3.5399528582162247, + "tokens_seen": 1389953024 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029236710130391174, + "loss": 2.6793, + "theoretical_loss": 3.5399379265111683, + "tokens_seen": 1390018560 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923570712136409, + "loss": 2.6415, + "theoretical_loss": 3.539922995707197, + "tokens_seen": 1390084096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923470411233701, + "loss": 2.5822, + "theoretical_loss": 3.5399080658042132, + "tokens_seen": 1390149632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923370110330993, + "loss": 2.6738, + "theoretical_loss": 3.53989313680212, + "tokens_seen": 1390215168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923269809428285, + "loss": 2.5026, + "theoretical_loss": 3.5398782087008214, + "tokens_seen": 1390280704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029231695085255765, + "loss": 2.5785, + "theoretical_loss": 3.53986328150022, + "tokens_seen": 1390346240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923069207622869, + "loss": 2.6385, + "theoretical_loss": 3.539848355200219, + "tokens_seen": 1390411776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029229689067201606, + "loss": 2.5758, + "theoretical_loss": 3.539833429800722, + "tokens_seen": 1390477312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029228686058174524, + "loss": 2.6024, + "theoretical_loss": 3.539818505301632, + "tokens_seen": 1390542848 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922768304914744, + "loss": 2.8159, + "theoretical_loss": 3.539803581702852, + "tokens_seen": 1390608384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922668004012036, + "loss": 2.589, + "theoretical_loss": 3.5397886590042855, + "tokens_seen": 1390673920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922567703109328, + "loss": 2.5165, + "theoretical_loss": 3.539773737205836, + "tokens_seen": 1390739456 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292246740220662, + "loss": 2.3329, + "theoretical_loss": 3.5397588163074065, + "tokens_seen": 1390804992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029223671013039115, + "loss": 2.5392, + "theoretical_loss": 3.5397438963089005, + "tokens_seen": 1390870528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922266800401204, + "loss": 2.6437, + "theoretical_loss": 3.5397289772102205, + "tokens_seen": 1390936064 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 766696, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6347217559814453, + "objective/train/theoretical_loss": 3.5397140590112715, + "objective/train/tokens_used": 1411461600, + "theoretical_loss": 3.5397140590112715, + "tokens_seen": 1391001600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922166499498495, + "loss": 2.618, + "theoretical_loss": 3.5397140590112715, + "tokens_seen": 1391001600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029220661985957875, + "loss": 2.6174, + "theoretical_loss": 3.5396991417119557, + "tokens_seen": 1391067136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921965897693079, + "loss": 2.5041, + "theoretical_loss": 3.5396842253121763, + "tokens_seen": 1391132672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921865596790371, + "loss": 2.5927, + "theoretical_loss": 3.539669309811837, + "tokens_seen": 1391198208 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921765295887663, + "loss": 2.6222, + "theoretical_loss": 3.5396543952108415, + "tokens_seen": 1391263744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029216649949849547, + "loss": 2.5897, + "theoretical_loss": 3.5396394815090932, + "tokens_seen": 1391329280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029215646940822465, + "loss": 2.7557, + "theoretical_loss": 3.5396245687064947, + "tokens_seen": 1391394816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921464393179539, + "loss": 2.5768, + "theoretical_loss": 3.5396096568029503, + "tokens_seen": 1391460352 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292136409227683, + "loss": 2.5327, + "theoretical_loss": 3.5395947457983636, + "tokens_seen": 1391525888 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029212637913741225, + "loss": 2.5041, + "theoretical_loss": 3.539579835692637, + "tokens_seen": 1391591424 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029211634904714143, + "loss": 2.5025, + "theoretical_loss": 3.5395649264856752, + "tokens_seen": 1391656960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921063189568706, + "loss": 2.6626, + "theoretical_loss": 3.5395500181773807, + "tokens_seen": 1391722496 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920962888665998, + "loss": 2.5822, + "theoretical_loss": 3.539535110767658, + "tokens_seen": 1391788032 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292086258776329, + "loss": 2.6969, + "theoretical_loss": 3.5395202042564096, + "tokens_seen": 1391853568 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029207622868605815, + "loss": 2.4481, + "theoretical_loss": 3.5395052986435394, + "tokens_seen": 1391919104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920661985957874, + "loss": 2.467, + "theoretical_loss": 3.5394903939289515, + "tokens_seen": 1391984640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920561685055165, + "loss": 2.4963, + "theoretical_loss": 3.539475490112549, + "tokens_seen": 1392050176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029204613841524575, + "loss": 2.6307, + "theoretical_loss": 3.539460587194236, + "tokens_seen": 1392115712 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029203610832497493, + "loss": 2.6562, + "theoretical_loss": 3.539445685173915, + "tokens_seen": 1392181248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920260782347041, + "loss": 2.4054, + "theoretical_loss": 3.5394307840514907, + "tokens_seen": 1392246784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029201604814443335, + "loss": 2.7607, + "theoretical_loss": 3.5394158838268663, + "tokens_seen": 1392312320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920060180541625, + "loss": 2.5339, + "theoretical_loss": 3.5394009844999457, + "tokens_seen": 1392377856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919959879638917, + "loss": 2.8024, + "theoretical_loss": 3.539386086070632, + "tokens_seen": 1392443392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919859578736209, + "loss": 2.6481, + "theoretical_loss": 3.5393711885388295, + "tokens_seen": 1392508928 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919759277833501, + "loss": 2.7028, + "theoretical_loss": 3.5393562919044417, + "tokens_seen": 1392574464 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 767843, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7980923652648926, + "objective/train/theoretical_loss": 3.539341396167372, + "objective/train/tokens_used": 1413100000, + "theoretical_loss": 3.539341396167372, + "tokens_seen": 1392640000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029196589769307926, + "loss": 2.6455, + "theoretical_loss": 3.539341396167372, + "tokens_seen": 1392640000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029195586760280844, + "loss": 2.4258, + "theoretical_loss": 3.5393265013275244, + "tokens_seen": 1392705536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919458375125376, + "loss": 2.6302, + "theoretical_loss": 3.5393116073848025, + "tokens_seen": 1392771072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029193580742226685, + "loss": 2.6789, + "theoretical_loss": 3.5392967143391107, + "tokens_seen": 1392836608 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291925777331996, + "loss": 2.4883, + "theoretical_loss": 3.539281822190352, + "tokens_seen": 1392902144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919157472417252, + "loss": 2.4174, + "theoretical_loss": 3.53926693093843, + "tokens_seen": 1392967680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029190571715145434, + "loss": 2.7658, + "theoretical_loss": 3.5392520405832495, + "tokens_seen": 1393033216 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918956870611836, + "loss": 2.7246, + "theoretical_loss": 3.5392371511247136, + "tokens_seen": 1393098752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029188565697091276, + "loss": 2.5244, + "theoretical_loss": 3.5392222625627263, + "tokens_seen": 1393164288 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029187562688064194, + "loss": 2.5479, + "theoretical_loss": 3.5392073748971913, + "tokens_seen": 1393229824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918655967903711, + "loss": 2.537, + "theoretical_loss": 3.5391924881280126, + "tokens_seen": 1393295360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918555667001003, + "loss": 2.5608, + "theoretical_loss": 3.5391776022550943, + "tokens_seen": 1393360896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918455366098295, + "loss": 2.7187, + "theoretical_loss": 3.5391627172783395, + "tokens_seen": 1393426432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918355065195587, + "loss": 2.633, + "theoretical_loss": 3.539147833197653, + "tokens_seen": 1393491968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029182547642928785, + "loss": 2.5427, + "theoretical_loss": 3.539132950012938, + "tokens_seen": 1393557504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918154463390171, + "loss": 2.6454, + "theoretical_loss": 3.5391180677240994, + "tokens_seen": 1393623040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029180541624874626, + "loss": 2.4717, + "theoretical_loss": 3.5391031863310403, + "tokens_seen": 1393688576 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029179538615847544, + "loss": 2.6568, + "theoretical_loss": 3.5390883058336655, + "tokens_seen": 1393754112 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917853560682046, + "loss": 2.5793, + "theoretical_loss": 3.5390734262318775, + "tokens_seen": 1393819648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917753259779338, + "loss": 2.7018, + "theoretical_loss": 3.539058547525581, + "tokens_seen": 1393885184 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291765295887663, + "loss": 2.5705, + "theoretical_loss": 3.5390436697146814, + "tokens_seen": 1393950720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917552657973922, + "loss": 2.7231, + "theoretical_loss": 3.5390287927990807, + "tokens_seen": 1394016256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029174523570712135, + "loss": 2.607, + "theoretical_loss": 3.539013916778684, + "tokens_seen": 1394081792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917352056168506, + "loss": 2.7236, + "theoretical_loss": 3.538999041653395, + "tokens_seen": 1394147328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917251755265797, + "loss": 2.5751, + "theoretical_loss": 3.5389841674231177, + "tokens_seen": 1394212864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 768328, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.26961350440979, + "objective/train/theoretical_loss": 3.5389692940877566, + "objective/train/tokens_used": 1414738400, + "theoretical_loss": 3.5389692940877566, + "tokens_seen": 1394278400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029171514543630895, + "loss": 2.4375, + "theoretical_loss": 3.5389692940877566, + "tokens_seen": 1394278400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917051153460381, + "loss": 2.6149, + "theoretical_loss": 3.5389544216472153, + "tokens_seen": 1394343936 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916950852557673, + "loss": 2.6194, + "theoretical_loss": 3.5389395501013983, + "tokens_seen": 1394409472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916850551654965, + "loss": 2.7015, + "theoretical_loss": 3.5389246794502096, + "tokens_seen": 1394475008 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029167502507522567, + "loss": 2.9316, + "theoretical_loss": 3.5389098096935534, + "tokens_seen": 1394540544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029166499498495485, + "loss": 2.5839, + "theoretical_loss": 3.5388949408313337, + "tokens_seen": 1394606080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916549648946841, + "loss": 2.7859, + "theoretical_loss": 3.5388800728634546, + "tokens_seen": 1394671616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916449348044132, + "loss": 2.5476, + "theoretical_loss": 3.538865205789821, + "tokens_seen": 1394737152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029163490471414245, + "loss": 2.5964, + "theoretical_loss": 3.538850339610336, + "tokens_seen": 1394802688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029162487462387163, + "loss": 2.5813, + "theoretical_loss": 3.5388354743249044, + "tokens_seen": 1394868224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916148445336008, + "loss": 2.5654, + "theoretical_loss": 3.5388206099334303, + "tokens_seen": 1394933760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029160481444333, + "loss": 2.2822, + "theoretical_loss": 3.5388057464358185, + "tokens_seen": 1394999296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915947843530592, + "loss": 2.5525, + "theoretical_loss": 3.5387908838319726, + "tokens_seen": 1395064832 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029158475426278835, + "loss": 2.6056, + "theoretical_loss": 3.5387760221217968, + "tokens_seen": 1395130368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915747241725176, + "loss": 2.648, + "theoretical_loss": 3.538761161305196, + "tokens_seen": 1395195904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915646940822467, + "loss": 2.6446, + "theoretical_loss": 3.5387463013820737, + "tokens_seen": 1395261440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029155466399197595, + "loss": 2.778, + "theoretical_loss": 3.538731442352335, + "tokens_seen": 1395326976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915446339017051, + "loss": 2.6212, + "theoretical_loss": 3.5387165842158836, + "tokens_seen": 1395392512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915346038114343, + "loss": 2.6653, + "theoretical_loss": 3.538701726972625, + "tokens_seen": 1395458048 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915245737211635, + "loss": 2.6928, + "theoretical_loss": 3.5386868706224615, + "tokens_seen": 1395523584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915145436308927, + "loss": 2.7496, + "theoretical_loss": 3.5386720151652997, + "tokens_seen": 1395589120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029150451354062186, + "loss": 2.4478, + "theoretical_loss": 3.5386571606010424, + "tokens_seen": 1395654656 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914944834503511, + "loss": 2.743, + "theoretical_loss": 3.538642306929595, + "tokens_seen": 1395720192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914844533600802, + "loss": 2.8987, + "theoretical_loss": 3.538627454150861, + "tokens_seen": 1395785728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029147442326980946, + "loss": 2.6115, + "theoretical_loss": 3.5386126022647457, + "tokens_seen": 1395851264 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 769315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.528290033340454, + "objective/train/theoretical_loss": 3.538597751271153, + "objective/train/tokens_used": 1416376800, + "theoretical_loss": 3.538597751271153, + "tokens_seen": 1395916800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914643931795386, + "loss": 2.5921, + "theoretical_loss": 3.538597751271153, + "tokens_seen": 1395916800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914543630892678, + "loss": 2.6469, + "theoretical_loss": 3.538582901169988, + "tokens_seen": 1395982336 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291444332998997, + "loss": 2.7489, + "theoretical_loss": 3.538568051961154, + "tokens_seen": 1396047872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914343029087262, + "loss": 2.7785, + "theoretical_loss": 3.5385532036445566, + "tokens_seen": 1396113408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029142427281845536, + "loss": 2.6316, + "theoretical_loss": 3.5385383562201005, + "tokens_seen": 1396178944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029141424272818454, + "loss": 2.4348, + "theoretical_loss": 3.538523509687689, + "tokens_seen": 1396244480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914042126379137, + "loss": 2.5896, + "theoretical_loss": 3.538508664047227, + "tokens_seen": 1396310016 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029139418254764296, + "loss": 2.6467, + "theoretical_loss": 3.53849381929862, + "tokens_seen": 1396375552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913841524573721, + "loss": 2.577, + "theoretical_loss": 3.5384789754417723, + "tokens_seen": 1396441088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913741223671013, + "loss": 2.5934, + "theoretical_loss": 3.5384641324765873, + "tokens_seen": 1396506624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029136409227683045, + "loss": 2.8131, + "theoretical_loss": 3.538449290402971, + "tokens_seen": 1396572160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913540621865597, + "loss": 2.5786, + "theoretical_loss": 3.538434449220827, + "tokens_seen": 1396637696 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029134403209628886, + "loss": 2.612, + "theoretical_loss": 3.5384196089300604, + "tokens_seen": 1396703232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029133400200601805, + "loss": 2.2821, + "theoretical_loss": 3.538404769530576, + "tokens_seen": 1396768768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913239719157472, + "loss": 2.9357, + "theoretical_loss": 3.5383899310222775, + "tokens_seen": 1396834304 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029131394182547646, + "loss": 2.7306, + "theoretical_loss": 3.5383750934050715, + "tokens_seen": 1396899840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913039117352056, + "loss": 2.6315, + "theoretical_loss": 3.538360256678861, + "tokens_seen": 1396965376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912938816449348, + "loss": 2.7702, + "theoretical_loss": 3.5383454208435507, + "tokens_seen": 1397030912 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291283851554664, + "loss": 2.5113, + "theoretical_loss": 3.5383305858990464, + "tokens_seen": 1397096448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912738214643932, + "loss": 2.7669, + "theoretical_loss": 3.538315751845252, + "tokens_seen": 1397161984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912637913741224, + "loss": 2.7036, + "theoretical_loss": 3.5383009186820726, + "tokens_seen": 1397227520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029125376128385155, + "loss": 2.7275, + "theoretical_loss": 3.538286086409413, + "tokens_seen": 1397293056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912437311935808, + "loss": 2.5954, + "theoretical_loss": 3.5382712550271775, + "tokens_seen": 1397358592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912337011033099, + "loss": 2.4268, + "theoretical_loss": 3.5382564245352714, + "tokens_seen": 1397424128 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029122367101303915, + "loss": 2.494, + "theoretical_loss": 3.5382415949335995, + "tokens_seen": 1397489664 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 769972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.501089096069336, + "objective/train/theoretical_loss": 3.538226766222066, + "objective/train/tokens_used": 1418015200, + "theoretical_loss": 3.538226766222066, + "tokens_seen": 1397555200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029121364092276833, + "loss": 2.532, + "theoretical_loss": 3.538226766222066, + "tokens_seen": 1397555200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912036108324975, + "loss": 2.5163, + "theoretical_loss": 3.5382119384005764, + "tokens_seen": 1397620736 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911935807422267, + "loss": 2.6278, + "theoretical_loss": 3.538197111469035, + "tokens_seen": 1397686272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029118355065195587, + "loss": 2.5544, + "theoretical_loss": 3.5381822854273475, + "tokens_seen": 1397751808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029117352056168505, + "loss": 2.9922, + "theoretical_loss": 3.538167460275418, + "tokens_seen": 1397817344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911634904714143, + "loss": 2.6333, + "theoretical_loss": 3.5381526360131517, + "tokens_seen": 1397882880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911534603811434, + "loss": 2.4192, + "theoretical_loss": 3.5381378126404535, + "tokens_seen": 1397948416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029114343029087265, + "loss": 2.4743, + "theoretical_loss": 3.5381229901572278, + "tokens_seen": 1398013952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029113340020060183, + "loss": 2.5949, + "theoretical_loss": 3.5381081685633804, + "tokens_seen": 1398079488 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291123370110331, + "loss": 2.545, + "theoretical_loss": 3.538093347858816, + "tokens_seen": 1398145024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911133400200602, + "loss": 2.8087, + "theoretical_loss": 3.538078528043439, + "tokens_seen": 1398210560 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911033099297894, + "loss": 2.519, + "theoretical_loss": 3.538063709117155, + "tokens_seen": 1398276096 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029109327983951855, + "loss": 2.6899, + "theoretical_loss": 3.538048891079869, + "tokens_seen": 1398341632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910832497492478, + "loss": 2.4182, + "theoretical_loss": 3.5380340739314855, + "tokens_seen": 1398407168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910732196589769, + "loss": 2.6658, + "theoretical_loss": 3.53801925767191, + "tokens_seen": 1398472704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029106318956870615, + "loss": 2.8556, + "theoretical_loss": 3.538004442301047, + "tokens_seen": 1398538240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910531594784353, + "loss": 2.6686, + "theoretical_loss": 3.537989627818802, + "tokens_seen": 1398603776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910431293881645, + "loss": 2.7685, + "theoretical_loss": 3.5379748142250804, + "tokens_seen": 1398669312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910330992978937, + "loss": 2.7405, + "theoretical_loss": 3.537960001519786, + "tokens_seen": 1398734848 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910230692076229, + "loss": 2.5712, + "theoretical_loss": 3.5379451897028256, + "tokens_seen": 1398800384 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029101303911735206, + "loss": 2.6448, + "theoretical_loss": 3.537930378774103, + "tokens_seen": 1398865920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910030090270813, + "loss": 2.484, + "theoretical_loss": 3.537915568733524, + "tokens_seen": 1398931456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909929789368104, + "loss": 2.8006, + "theoretical_loss": 3.5379007595809933, + "tokens_seen": 1398996992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029098294884653966, + "loss": 2.3332, + "theoretical_loss": 3.5378859513164165, + "tokens_seen": 1399062528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909729187562688, + "loss": 2.6569, + "theoretical_loss": 3.5378711439396984, + "tokens_seen": 1399128064 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 771272, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4476327896118164, + "objective/train/theoretical_loss": 3.5378563374507443, + "objective/train/tokens_used": 1419653600, + "theoretical_loss": 3.5378563374507443, + "tokens_seen": 1399193600 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290962888665998, + "loss": 2.5499, + "theoretical_loss": 3.5378563374507443, + "tokens_seen": 1399193600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909528585757272, + "loss": 2.6638, + "theoretical_loss": 3.53784153184946, + "tokens_seen": 1399259136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909428284854564, + "loss": 2.605, + "theoretical_loss": 3.5378267271357493, + "tokens_seen": 1399324672 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029093279839518556, + "loss": 2.5923, + "theoretical_loss": 3.537811923309518, + "tokens_seen": 1399390208 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029092276830491474, + "loss": 2.56, + "theoretical_loss": 3.537797120370673, + "tokens_seen": 1399455744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909127382146439, + "loss": 2.7205, + "theoretical_loss": 3.537782318319117, + "tokens_seen": 1399521280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029090270812437316, + "loss": 2.4415, + "theoretical_loss": 3.537767517154757, + "tokens_seen": 1399586816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908926780341023, + "loss": 2.7773, + "theoretical_loss": 3.537752716877497, + "tokens_seen": 1399652352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908826479438315, + "loss": 2.6969, + "theoretical_loss": 3.5377379174872434, + "tokens_seen": 1399717888 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029087261785356065, + "loss": 2.6716, + "theoretical_loss": 3.5377231189839016, + "tokens_seen": 1399783424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908625877632899, + "loss": 2.5032, + "theoretical_loss": 3.5377083213673757, + "tokens_seen": 1399848960 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029085255767301906, + "loss": 2.6298, + "theoretical_loss": 3.5376935246375716, + "tokens_seen": 1399914496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029084252758274825, + "loss": 2.6113, + "theoretical_loss": 3.537678728794396, + "tokens_seen": 1399980032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908324974924774, + "loss": 2.6666, + "theoretical_loss": 3.5376639338377522, + "tokens_seen": 1400045568 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029082246740220666, + "loss": 2.436, + "theoretical_loss": 3.537649139767547, + "tokens_seen": 1400111104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908124373119358, + "loss": 2.594, + "theoretical_loss": 3.5376343465836846, + "tokens_seen": 1400176640 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290802407221665, + "loss": 2.4731, + "theoretical_loss": 3.5376195542860716, + "tokens_seen": 1400242176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029079237713139415, + "loss": 2.7171, + "theoretical_loss": 3.5376047628746132, + "tokens_seen": 1400307712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907823470411234, + "loss": 2.5599, + "theoretical_loss": 3.537589972349214, + "tokens_seen": 1400373248 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029077231695085257, + "loss": 2.6427, + "theoretical_loss": 3.53757518270978, + "tokens_seen": 1400438784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029076228686058175, + "loss": 2.5244, + "theoretical_loss": 3.5375603939562175, + "tokens_seen": 1400504320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029075225677031093, + "loss": 2.5821, + "theoretical_loss": 3.5375456060884307, + "tokens_seen": 1400569856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907422266800401, + "loss": 2.3923, + "theoretical_loss": 3.5375308191063257, + "tokens_seen": 1400635392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907321965897693, + "loss": 2.5818, + "theoretical_loss": 3.537516033009808, + "tokens_seen": 1400700928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029072216649949853, + "loss": 2.6046, + "theoretical_loss": 3.537501247798783, + "tokens_seen": 1400766464 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 771955, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5957698822021484, + "objective/train/theoretical_loss": 3.537486463473156, + "objective/train/tokens_used": 1421292000, + "theoretical_loss": 3.537486463473156, + "tokens_seen": 1400832000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029071213640922765, + "loss": 2.6511, + "theoretical_loss": 3.537486463473156, + "tokens_seen": 1400832000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907021063189569, + "loss": 2.589, + "theoretical_loss": 3.5374716800328336, + "tokens_seen": 1400897536 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290692076228686, + "loss": 2.6191, + "theoretical_loss": 3.5374568974777203, + "tokens_seen": 1400963072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029068204613841525, + "loss": 2.4094, + "theoretical_loss": 3.537442115807722, + "tokens_seen": 1401028608 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029067201604814443, + "loss": 2.8195, + "theoretical_loss": 3.537427335022744, + "tokens_seen": 1401094144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906619859578736, + "loss": 2.4319, + "theoretical_loss": 3.5374125551226925, + "tokens_seen": 1401159680 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906519558676028, + "loss": 2.224, + "theoretical_loss": 3.537397776107473, + "tokens_seen": 1401225216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029064192577733203, + "loss": 2.5957, + "theoretical_loss": 3.537382997976991, + "tokens_seen": 1401290752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029063189568706116, + "loss": 2.5608, + "theoretical_loss": 3.5373682207311523, + "tokens_seen": 1401356288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906218655967904, + "loss": 2.8076, + "theoretical_loss": 3.537353444369862, + "tokens_seen": 1401421824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906118355065195, + "loss": 2.4087, + "theoretical_loss": 3.537338668893027, + "tokens_seen": 1401487360 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029060180541624876, + "loss": 2.5909, + "theoretical_loss": 3.5373238943005516, + "tokens_seen": 1401552896 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029059177532597794, + "loss": 2.5853, + "theoretical_loss": 3.5373091205923424, + "tokens_seen": 1401618432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905817452357071, + "loss": 2.6245, + "theoretical_loss": 3.537294347768305, + "tokens_seen": 1401683968 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905717151454363, + "loss": 2.5226, + "theoretical_loss": 3.5372795758283453, + "tokens_seen": 1401749504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905616850551655, + "loss": 2.5014, + "theoretical_loss": 3.5372648047723683, + "tokens_seen": 1401815040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029055165496489466, + "loss": 2.711, + "theoretical_loss": 3.5372500346002806, + "tokens_seen": 1401880576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905416248746239, + "loss": 2.7036, + "theoretical_loss": 3.5372352653119883, + "tokens_seen": 1401946112 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905315947843531, + "loss": 2.4211, + "theoretical_loss": 3.5372204969073957, + "tokens_seen": 1402011648 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029052156469408226, + "loss": 2.6379, + "theoretical_loss": 3.5372057293864096, + "tokens_seen": 1402077184 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905115346038115, + "loss": 2.7198, + "theoretical_loss": 3.5371909627489364, + "tokens_seen": 1402142720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905015045135406, + "loss": 2.7426, + "theoretical_loss": 3.5371761969948805, + "tokens_seen": 1402208256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029049147442326986, + "loss": 2.5106, + "theoretical_loss": 3.537161432124149, + "tokens_seen": 1402273792 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290481444332999, + "loss": 2.9995, + "theoretical_loss": 3.5371466681366477, + "tokens_seen": 1402339328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904714142427282, + "loss": 2.4158, + "theoretical_loss": 3.5371319050322816, + "tokens_seen": 1402404864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 773050, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4128201007843018, + "objective/train/theoretical_loss": 3.5371171428109576, + "objective/train/tokens_used": 1422930400, + "theoretical_loss": 3.5371171428109576, + "tokens_seen": 1402470400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904613841524574, + "loss": 2.5512, + "theoretical_loss": 3.5371171428109576, + "tokens_seen": 1402470400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002904513540621866, + "loss": 2.5093, + "theoretical_loss": 3.5371023814725806, + "tokens_seen": 1402535936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029044132397191576, + "loss": 2.6765, + "theoretical_loss": 3.537087621017058, + "tokens_seen": 1402601472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029043129388164494, + "loss": 2.6637, + "theoretical_loss": 3.537072861444294, + "tokens_seen": 1402667008 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002904212637913741, + "loss": 2.6707, + "theoretical_loss": 3.5370581027541963, + "tokens_seen": 1402732544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029041123370110336, + "loss": 2.705, + "theoretical_loss": 3.537043344946669, + "tokens_seen": 1402798080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002904012036108325, + "loss": 2.3418, + "theoretical_loss": 3.5370285880216197, + "tokens_seen": 1402863616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903911735205617, + "loss": 2.6118, + "theoretical_loss": 3.5370138319789537, + "tokens_seen": 1402929152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029038114343029085, + "loss": 2.4865, + "theoretical_loss": 3.536999076818577, + "tokens_seen": 1402994688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903711133400201, + "loss": 2.3003, + "theoretical_loss": 3.536984322540396, + "tokens_seen": 1403060224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029036108324974926, + "loss": 2.605, + "theoretical_loss": 3.5369695691443166, + "tokens_seen": 1403125760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029035105315947845, + "loss": 2.599, + "theoretical_loss": 3.5369548166302445, + "tokens_seen": 1403191296 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903410230692076, + "loss": 2.5252, + "theoretical_loss": 3.5369400649980864, + "tokens_seen": 1403256832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029033099297893686, + "loss": 2.8072, + "theoretical_loss": 3.536925314247748, + "tokens_seen": 1403322368 + }, + { + "epoch": 0.43, + "learning_rate": 0.000290320962888666, + "loss": 2.5232, + "theoretical_loss": 3.536910564379135, + "tokens_seen": 1403387904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903109327983952, + "loss": 2.4439, + "theoretical_loss": 3.536895815392154, + "tokens_seen": 1403453440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029030090270812435, + "loss": 2.7899, + "theoretical_loss": 3.536881067286712, + "tokens_seen": 1403518976 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902908726178536, + "loss": 2.8269, + "theoretical_loss": 3.5368663200627135, + "tokens_seen": 1403584512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029028084252758277, + "loss": 2.5834, + "theoretical_loss": 3.5368515737200656, + "tokens_seen": 1403650048 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029027081243731195, + "loss": 2.5682, + "theoretical_loss": 3.5368368282586746, + "tokens_seen": 1403715584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029026078234704113, + "loss": 2.5021, + "theoretical_loss": 3.5368220836784463, + "tokens_seen": 1403781120 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902507522567703, + "loss": 2.5592, + "theoretical_loss": 3.536807339979287, + "tokens_seen": 1403846656 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902407221664995, + "loss": 2.4743, + "theoretical_loss": 3.536792597161103, + "tokens_seen": 1403912192 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029023069207622873, + "loss": 2.5822, + "theoretical_loss": 3.5367778552238005, + "tokens_seen": 1403977728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029022066198595785, + "loss": 2.6146, + "theoretical_loss": 3.536763114167286, + "tokens_seen": 1404043264 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 774221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6937434673309326, + "objective/train/theoretical_loss": 3.5367483739914647, + "objective/train/tokens_used": 1424568800, + "theoretical_loss": 3.5367483739914647, + "tokens_seen": 1404108800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902106318956871, + "loss": 2.6184, + "theoretical_loss": 3.5367483739914647, + "tokens_seen": 1404108800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902006018054162, + "loss": 2.6297, + "theoretical_loss": 3.536733634696245, + "tokens_seen": 1404174336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029019057171514545, + "loss": 2.3651, + "theoretical_loss": 3.536718896281531, + "tokens_seen": 1404239872 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029018054162487463, + "loss": 2.6208, + "theoretical_loss": 3.53670415874723, + "tokens_seen": 1404305408 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901705115346038, + "loss": 2.5992, + "theoretical_loss": 3.5366894220932483, + "tokens_seen": 1404370944 + }, + { + "epoch": 0.43, + "learning_rate": 0.000290160481444333, + "loss": 2.4126, + "theoretical_loss": 3.536674686319492, + "tokens_seen": 1404436480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029015045135406223, + "loss": 2.4655, + "theoretical_loss": 3.5366599514258676, + "tokens_seen": 1404502016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029014042126379136, + "loss": 2.8439, + "theoretical_loss": 3.536645217412282, + "tokens_seen": 1404567552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901303911735206, + "loss": 2.672, + "theoretical_loss": 3.5366304842786405, + "tokens_seen": 1404633088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901203610832497, + "loss": 2.377, + "theoretical_loss": 3.5366157520248507, + "tokens_seen": 1404698624 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029011033099297896, + "loss": 2.5877, + "theoretical_loss": 3.5366010206508176, + "tokens_seen": 1404764160 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029010030090270814, + "loss": 2.4964, + "theoretical_loss": 3.536586290156449, + "tokens_seen": 1404829696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900902708124373, + "loss": 2.643, + "theoretical_loss": 3.5365715605416503, + "tokens_seen": 1404895232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900802407221665, + "loss": 2.7111, + "theoretical_loss": 3.5365568318063287, + "tokens_seen": 1404960768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900702106318957, + "loss": 2.6959, + "theoretical_loss": 3.5365421039503904, + "tokens_seen": 1405026304 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029006018054162486, + "loss": 2.6788, + "theoretical_loss": 3.5365273769737415, + "tokens_seen": 1405091840 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900501504513541, + "loss": 2.7087, + "theoretical_loss": 3.536512650876289, + "tokens_seen": 1405157376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900401203610832, + "loss": 2.5697, + "theoretical_loss": 3.5364979256579394, + "tokens_seen": 1405222912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029003009027081246, + "loss": 2.5978, + "theoretical_loss": 3.5364832013185996, + "tokens_seen": 1405288448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900200601805416, + "loss": 2.6004, + "theoretical_loss": 3.5364684778581745, + "tokens_seen": 1405353984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900100300902708, + "loss": 2.4113, + "theoretical_loss": 3.5364537552765727, + "tokens_seen": 1405419520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029, + "loss": 2.6976, + "theoretical_loss": 3.536439033573699, + "tokens_seen": 1405485056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899899699097292, + "loss": 2.7273, + "theoretical_loss": 3.5364243127494612, + "tokens_seen": 1405550592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028997993981945836, + "loss": 2.7515, + "theoretical_loss": 3.536409592803766, + "tokens_seen": 1405616128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899699097291876, + "loss": 2.4516, + "theoretical_loss": 3.536394873736519, + "tokens_seen": 1405681664 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 774861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.848618268966675, + "objective/train/theoretical_loss": 3.5363801555476275, + "objective/train/tokens_used": 1426207200, + "theoretical_loss": 3.5363801555476275, + "tokens_seen": 1405747200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899598796389167, + "loss": 2.5522, + "theoretical_loss": 3.5363801555476275, + "tokens_seen": 1405747200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028994984954864596, + "loss": 2.5532, + "theoretical_loss": 3.536365438236998, + "tokens_seen": 1405812736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899398194583751, + "loss": 2.6049, + "theoretical_loss": 3.536350721804537, + "tokens_seen": 1405878272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899297893681043, + "loss": 2.6394, + "theoretical_loss": 3.5363360062501514, + "tokens_seen": 1405943808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899197592778335, + "loss": 2.5589, + "theoretical_loss": 3.536321291573748, + "tokens_seen": 1406009344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899097291875627, + "loss": 2.7436, + "theoretical_loss": 3.5363065777752327, + "tokens_seen": 1406074880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028989969909729187, + "loss": 2.683, + "theoretical_loss": 3.5362918648545136, + "tokens_seen": 1406140416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028988966900702105, + "loss": 2.6996, + "theoretical_loss": 3.536277152811496, + "tokens_seen": 1406205952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028987963891675023, + "loss": 2.5788, + "theoretical_loss": 3.5362624416460875, + "tokens_seen": 1406271488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028986960882647946, + "loss": 2.622, + "theoretical_loss": 3.536247731358195, + "tokens_seen": 1406337024 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898595787362086, + "loss": 2.535, + "theoretical_loss": 3.536233021947724, + "tokens_seen": 1406402560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898495486459378, + "loss": 2.4694, + "theoretical_loss": 3.5362183134145826, + "tokens_seen": 1406468096 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289839518555667, + "loss": 2.6784, + "theoretical_loss": 3.5362036057586774, + "tokens_seen": 1406533632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898294884653962, + "loss": 2.5906, + "theoretical_loss": 3.5361888989799146, + "tokens_seen": 1406599168 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028981945837512537, + "loss": 2.5658, + "theoretical_loss": 3.536174193078202, + "tokens_seen": 1406664704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028980942828485455, + "loss": 2.4758, + "theoretical_loss": 3.5361594880534453, + "tokens_seen": 1406730240 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028979939819458373, + "loss": 2.5771, + "theoretical_loss": 3.536144783905552, + "tokens_seen": 1406795776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028978936810431297, + "loss": 2.5072, + "theoretical_loss": 3.536130080634429, + "tokens_seen": 1406861312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028977933801404215, + "loss": 2.5125, + "theoretical_loss": 3.536115378239983, + "tokens_seen": 1406926848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028976930792377133, + "loss": 2.4741, + "theoretical_loss": 3.5361006767221204, + "tokens_seen": 1406992384 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897592778335005, + "loss": 2.764, + "theoretical_loss": 3.536085976080749, + "tokens_seen": 1407057920 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897492477432297, + "loss": 2.4863, + "theoretical_loss": 3.5360712763157753, + "tokens_seen": 1407123456 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028973921765295893, + "loss": 2.6076, + "theoretical_loss": 3.536056577427107, + "tokens_seen": 1407188992 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028972918756268805, + "loss": 2.6422, + "theoretical_loss": 3.5360418794146495, + "tokens_seen": 1407254528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897191574724173, + "loss": 2.6092, + "theoretical_loss": 3.5360271822783114, + "tokens_seen": 1407320064 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 776313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.495975971221924, + "objective/train/theoretical_loss": 3.536012486017998, + "objective/train/tokens_used": 1427845600, + "theoretical_loss": 3.536012486017998, + "tokens_seen": 1407385600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897091273821464, + "loss": 2.52, + "theoretical_loss": 3.536012486017998, + "tokens_seen": 1407385600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028969909729187565, + "loss": 2.4312, + "theoretical_loss": 3.5359977906336173, + "tokens_seen": 1407451136 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028968906720160483, + "loss": 2.7821, + "theoretical_loss": 3.5359830961250767, + "tokens_seen": 1407516672 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289679037111334, + "loss": 2.4085, + "theoretical_loss": 3.5359684024922826, + "tokens_seen": 1407582208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896690070210632, + "loss": 2.6334, + "theoretical_loss": 3.535953709735143, + "tokens_seen": 1407647744 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028965897693079243, + "loss": 2.561, + "theoretical_loss": 3.535939017853563, + "tokens_seen": 1407713280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028964894684052156, + "loss": 2.4042, + "theoretical_loss": 3.535924326847451, + "tokens_seen": 1407778816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896389167502508, + "loss": 2.6134, + "theoretical_loss": 3.5359096367167138, + "tokens_seen": 1407844352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896288866599799, + "loss": 2.4784, + "theoretical_loss": 3.5358949474612587, + "tokens_seen": 1407909888 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028961885656970916, + "loss": 2.5981, + "theoretical_loss": 3.535880259080993, + "tokens_seen": 1407975424 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028960882647943834, + "loss": 2.6256, + "theoretical_loss": 3.5358655715758225, + "tokens_seen": 1408040960 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895987963891675, + "loss": 2.5376, + "theoretical_loss": 3.5358508849456562, + "tokens_seen": 1408106496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895887662988967, + "loss": 2.4982, + "theoretical_loss": 3.5358361991904, + "tokens_seen": 1408172032 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895787362086259, + "loss": 2.5318, + "theoretical_loss": 3.535821514309962, + "tokens_seen": 1408237568 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028956870611835506, + "loss": 2.659, + "theoretical_loss": 3.5358068303042485, + "tokens_seen": 1408303104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895586760280843, + "loss": 2.5674, + "theoretical_loss": 3.535792147173167, + "tokens_seen": 1408368640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895486459378134, + "loss": 2.5131, + "theoretical_loss": 3.5357774649166247, + "tokens_seen": 1408434176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028953861584754266, + "loss": 2.7549, + "theoretical_loss": 3.535762783534529, + "tokens_seen": 1408499712 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895285857572718, + "loss": 2.7811, + "theoretical_loss": 3.5357481030267865, + "tokens_seen": 1408565248 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289518555667001, + "loss": 2.6589, + "theoretical_loss": 3.5357334233933058, + "tokens_seen": 1408630784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895085255767302, + "loss": 2.7636, + "theoretical_loss": 3.535718744633993, + "tokens_seen": 1408696320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894984954864594, + "loss": 2.5477, + "theoretical_loss": 3.535704066748755, + "tokens_seen": 1408761856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028948846539618856, + "loss": 2.5133, + "theoretical_loss": 3.5356893897375006, + "tokens_seen": 1408827392 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894784353059178, + "loss": 2.4739, + "theoretical_loss": 3.5356747136001356, + "tokens_seen": 1408892928 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894684052156469, + "loss": 2.5042, + "theoretical_loss": 3.5356600383365686, + "tokens_seen": 1408958464 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 776966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5918385982513428, + "objective/train/theoretical_loss": 3.5356453639467063, + "objective/train/tokens_used": 1429484000, + "theoretical_loss": 3.5356453639467063, + "tokens_seen": 1409024000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028945837512537616, + "loss": 2.4673, + "theoretical_loss": 3.5356453639467063, + "tokens_seen": 1409024000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894483450351053, + "loss": 2.7497, + "theoretical_loss": 3.5356306904304553, + "tokens_seen": 1409089536 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894383149448345, + "loss": 2.5532, + "theoretical_loss": 3.5356160177877247, + "tokens_seen": 1409155072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894282848545637, + "loss": 2.5717, + "theoretical_loss": 3.5356013460184204, + "tokens_seen": 1409220608 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894182547642929, + "loss": 2.551, + "theoretical_loss": 3.5355866751224507, + "tokens_seen": 1409286144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028940822467402207, + "loss": 2.685, + "theoretical_loss": 3.5355720050997226, + "tokens_seen": 1409351680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028939819458375125, + "loss": 2.6845, + "theoretical_loss": 3.535557335950143, + "tokens_seen": 1409417216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028938816449348043, + "loss": 2.8001, + "theoretical_loss": 3.5355426676736204, + "tokens_seen": 1409482752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028937813440320966, + "loss": 2.7927, + "theoretical_loss": 3.5355280002700615, + "tokens_seen": 1409548288 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893681043129388, + "loss": 2.6813, + "theoretical_loss": 3.535513333739374, + "tokens_seen": 1409613824 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289358074222668, + "loss": 2.5749, + "theoretical_loss": 3.5354986680814657, + "tokens_seen": 1409679360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893480441323972, + "loss": 2.6353, + "theoretical_loss": 3.5354840032962436, + "tokens_seen": 1409744896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893380140421264, + "loss": 2.6406, + "theoretical_loss": 3.5354693393836154, + "tokens_seen": 1409810432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028932798395185557, + "loss": 2.6618, + "theoretical_loss": 3.5354546763434884, + "tokens_seen": 1409875968 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028931795386158475, + "loss": 2.3186, + "theoretical_loss": 3.5354400141757703, + "tokens_seen": 1409941504 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028930792377131393, + "loss": 2.7146, + "theoretical_loss": 3.5354253528803685, + "tokens_seen": 1410007040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028929789368104317, + "loss": 2.638, + "theoretical_loss": 3.535410692457191, + "tokens_seen": 1410072576 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892878635907723, + "loss": 2.6464, + "theoretical_loss": 3.535396032906145, + "tokens_seen": 1410138112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028927783350050153, + "loss": 2.6176, + "theoretical_loss": 3.5353813742271383, + "tokens_seen": 1410203648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028926780341023066, + "loss": 2.5145, + "theoretical_loss": 3.5353667164200786, + "tokens_seen": 1410269184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892577733199599, + "loss": 2.8026, + "theoretical_loss": 3.5353520594848726, + "tokens_seen": 1410334720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892477432296891, + "loss": 2.3777, + "theoretical_loss": 3.5353374034214293, + "tokens_seen": 1410400256 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028923771313941825, + "loss": 2.6062, + "theoretical_loss": 3.535322748229656, + "tokens_seen": 1410465792 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028922768304914744, + "loss": 2.5894, + "theoretical_loss": 3.5353080939094594, + "tokens_seen": 1410531328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892176529588766, + "loss": 2.8566, + "theoretical_loss": 3.5352934404607477, + "tokens_seen": 1410596864 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 777918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2443416118621826, + "objective/train/theoretical_loss": 3.535278787883429, + "objective/train/tokens_used": 1431122400, + "theoretical_loss": 3.535278787883429, + "tokens_seen": 1410662400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892076228686058, + "loss": 2.479, + "theoretical_loss": 3.535278787883429, + "tokens_seen": 1410662400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028919759277833503, + "loss": 2.5928, + "theoretical_loss": 3.5352641361774104, + "tokens_seen": 1410727936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028918756268806416, + "loss": 2.4782, + "theoretical_loss": 3.5352494853426, + "tokens_seen": 1410793472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891775325977934, + "loss": 2.4867, + "theoretical_loss": 3.5352348353789065, + "tokens_seen": 1410859008 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891675025075226, + "loss": 2.7544, + "theoretical_loss": 3.535220186286235, + "tokens_seen": 1410924544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028915747241725176, + "loss": 2.5548, + "theoretical_loss": 3.535205538064496, + "tokens_seen": 1410990080 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028914744232698094, + "loss": 2.8223, + "theoretical_loss": 3.5351908907135954, + "tokens_seen": 1411055616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891374122367101, + "loss": 2.5629, + "theoretical_loss": 3.5351762442334422, + "tokens_seen": 1411121152 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891273821464393, + "loss": 2.6886, + "theoretical_loss": 3.5351615986239437, + "tokens_seen": 1411186688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028911735205616854, + "loss": 2.5453, + "theoretical_loss": 3.5351469538850075, + "tokens_seen": 1411252224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028910732196589766, + "loss": 2.746, + "theoretical_loss": 3.535132310016542, + "tokens_seen": 1411317760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890972918756269, + "loss": 2.6494, + "theoretical_loss": 3.5351176670184543, + "tokens_seen": 1411383296 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289087261785356, + "loss": 2.577, + "theoretical_loss": 3.535103024890653, + "tokens_seen": 1411448832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028907723169508526, + "loss": 2.6034, + "theoretical_loss": 3.5350883836330453, + "tokens_seen": 1411514368 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028906720160481444, + "loss": 2.4017, + "theoretical_loss": 3.53507374324554, + "tokens_seen": 1411579904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890571715145436, + "loss": 2.392, + "theoretical_loss": 3.5350591037280434, + "tokens_seen": 1411645440 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890471414242728, + "loss": 2.6675, + "theoretical_loss": 3.5350444650804658, + "tokens_seen": 1411710976 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289037111334002, + "loss": 2.4364, + "theoretical_loss": 3.5350298273027128, + "tokens_seen": 1411776512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890270812437312, + "loss": 2.5823, + "theoretical_loss": 3.5350151903946934, + "tokens_seen": 1411842048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890170511534604, + "loss": 2.5937, + "theoretical_loss": 3.5350005543563157, + "tokens_seen": 1411907584 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890070210631896, + "loss": 2.5991, + "theoretical_loss": 3.534985919187487, + "tokens_seen": 1411973120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028899699097291876, + "loss": 2.5725, + "theoretical_loss": 3.5349712848881163, + "tokens_seen": 1412038656 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288986960882648, + "loss": 2.5371, + "theoretical_loss": 3.5349566514581103, + "tokens_seen": 1412104192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889769307923771, + "loss": 2.5297, + "theoretical_loss": 3.534942018897378, + "tokens_seen": 1412169728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028896690070210636, + "loss": 2.6743, + "theoretical_loss": 3.534927387205827, + "tokens_seen": 1412235264 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 778542, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5615313053131104, + "objective/train/theoretical_loss": 3.534912756383366, + "objective/train/tokens_used": 1432760800, + "theoretical_loss": 3.534912756383366, + "tokens_seen": 1412300800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889568706118355, + "loss": 2.7307, + "theoretical_loss": 3.534912756383366, + "tokens_seen": 1412300800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889468405215647, + "loss": 2.5797, + "theoretical_loss": 3.534898126429902, + "tokens_seen": 1412366336 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889368104312939, + "loss": 2.4541, + "theoretical_loss": 3.534883497345344, + "tokens_seen": 1412431872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889267803410231, + "loss": 2.7214, + "theoretical_loss": 3.534868869129599, + "tokens_seen": 1412497408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028891675025075227, + "loss": 2.8409, + "theoretical_loss": 3.5348542417825763, + "tokens_seen": 1412562944 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028890672016048145, + "loss": 2.7457, + "theoretical_loss": 3.5348396153041834, + "tokens_seen": 1412628480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028889669007021063, + "loss": 2.613, + "theoretical_loss": 3.5348249896943287, + "tokens_seen": 1412694016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028888665997993986, + "loss": 2.5372, + "theoretical_loss": 3.5348103649529197, + "tokens_seen": 1412759552 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288876629889669, + "loss": 2.5855, + "theoretical_loss": 3.534795741079865, + "tokens_seen": 1412825088 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028886659979939823, + "loss": 2.4765, + "theoretical_loss": 3.534781118075073, + "tokens_seen": 1412890624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888565697091274, + "loss": 2.7239, + "theoretical_loss": 3.5347664959384515, + "tokens_seen": 1412956160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888465396188566, + "loss": 2.7046, + "theoretical_loss": 3.534751874669909, + "tokens_seen": 1413021696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028883650952858577, + "loss": 2.4911, + "theoretical_loss": 3.534737254269353, + "tokens_seen": 1413087232 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028882647943831495, + "loss": 2.6196, + "theoretical_loss": 3.534722634736693, + "tokens_seen": 1413152768 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028881644934804413, + "loss": 2.3404, + "theoretical_loss": 3.5347080160718356, + "tokens_seen": 1413218304 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028880641925777337, + "loss": 2.6794, + "theoretical_loss": 3.5346933982746904, + "tokens_seen": 1413283840 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887963891675025, + "loss": 2.6401, + "theoretical_loss": 3.534678781345166, + "tokens_seen": 1413349376 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028878635907723173, + "loss": 2.5482, + "theoretical_loss": 3.5346641652831687, + "tokens_seen": 1413414912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028877632898696086, + "loss": 2.6347, + "theoretical_loss": 3.5346495500886084, + "tokens_seen": 1413480448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887662988966901, + "loss": 2.843, + "theoretical_loss": 3.5346349357613924, + "tokens_seen": 1413545984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887562688064193, + "loss": 2.6257, + "theoretical_loss": 3.5346203223014303, + "tokens_seen": 1413611520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028874623871614845, + "loss": 2.9082, + "theoretical_loss": 3.5346057097086296, + "tokens_seen": 1413677056 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028873620862587764, + "loss": 2.5796, + "theoretical_loss": 3.534591097982899, + "tokens_seen": 1413742592 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887261785356068, + "loss": 2.6771, + "theoretical_loss": 3.534576487124146, + "tokens_seen": 1413808128 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288716148445336, + "loss": 2.6835, + "theoretical_loss": 3.5345618771322798, + "tokens_seen": 1413873664 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 779834, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2309916019439697, + "objective/train/theoretical_loss": 3.5345472680072083, + "objective/train/tokens_used": 1434399200, + "theoretical_loss": 3.5345472680072083, + "tokens_seen": 1413939200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028870611835506523, + "loss": 2.6106, + "theoretical_loss": 3.5345472680072083, + "tokens_seen": 1413939200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028869608826479436, + "loss": 2.6427, + "theoretical_loss": 3.534532659748841, + "tokens_seen": 1414004736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886860581745236, + "loss": 2.5728, + "theoretical_loss": 3.5345180523570847, + "tokens_seen": 1414070272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886760280842528, + "loss": 2.6264, + "theoretical_loss": 3.5345034458318487, + "tokens_seen": 1414135808 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028866599799398196, + "loss": 2.7622, + "theoretical_loss": 3.534488840173042, + "tokens_seen": 1414201344 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028865596790371114, + "loss": 2.7954, + "theoretical_loss": 3.534474235380572, + "tokens_seen": 1414266880 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886459378134403, + "loss": 2.5129, + "theoretical_loss": 3.5344596314543475, + "tokens_seen": 1414332416 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886359077231695, + "loss": 2.5018, + "theoretical_loss": 3.534445028394277, + "tokens_seen": 1414397952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028862587763289874, + "loss": 2.5364, + "theoretical_loss": 3.5344304262002693, + "tokens_seen": 1414463488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028861584754262786, + "loss": 2.824, + "theoretical_loss": 3.534415824872233, + "tokens_seen": 1414529024 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886058174523571, + "loss": 2.5825, + "theoretical_loss": 3.5344012244100758, + "tokens_seen": 1414594560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885957873620862, + "loss": 2.5794, + "theoretical_loss": 3.534386624813707, + "tokens_seen": 1414660096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028858575727181546, + "loss": 2.6594, + "theoretical_loss": 3.5343720260830347, + "tokens_seen": 1414725632 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028857572718154464, + "loss": 2.6169, + "theoretical_loss": 3.5343574282179677, + "tokens_seen": 1414791168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885656970912738, + "loss": 2.6218, + "theoretical_loss": 3.534342831218415, + "tokens_seen": 1414856704 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288555667001003, + "loss": 2.7485, + "theoretical_loss": 3.534328235084285, + "tokens_seen": 1414922240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885456369107322, + "loss": 2.6608, + "theoretical_loss": 3.534313639815485, + "tokens_seen": 1414987776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028853560682046137, + "loss": 2.5013, + "theoretical_loss": 3.534299045411925, + "tokens_seen": 1415053312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885255767301906, + "loss": 2.6526, + "theoretical_loss": 3.534284451873514, + "tokens_seen": 1415118848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028851554663991973, + "loss": 2.4369, + "theoretical_loss": 3.534269859200159, + "tokens_seen": 1415184384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028850551654964896, + "loss": 2.6079, + "theoretical_loss": 3.534255267391771, + "tokens_seen": 1415249920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028849548645937815, + "loss": 2.5427, + "theoretical_loss": 3.534240676448256, + "tokens_seen": 1415315456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884854563691073, + "loss": 2.7063, + "theoretical_loss": 3.5342260863695247, + "tokens_seen": 1415380992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884754262788365, + "loss": 2.7161, + "theoretical_loss": 3.534211497155485, + "tokens_seen": 1415446528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884653961885657, + "loss": 2.706, + "theoretical_loss": 3.534196908806046, + "tokens_seen": 1415512064 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 780488, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1402840614318848, + "objective/train/theoretical_loss": 3.534182321321116, + "objective/train/tokens_used": 1436037600, + "theoretical_loss": 3.534182321321116, + "tokens_seen": 1415577600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028845536609829487, + "loss": 2.5412, + "theoretical_loss": 3.534182321321116, + "tokens_seen": 1415577600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884453360080241, + "loss": 2.8081, + "theoretical_loss": 3.5341677347006035, + "tokens_seen": 1415643136 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028843530591775323, + "loss": 2.718, + "theoretical_loss": 3.5341531489444185, + "tokens_seen": 1415708672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028842527582748247, + "loss": 2.5688, + "theoretical_loss": 3.5341385640524683, + "tokens_seen": 1415774208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884152457372116, + "loss": 2.6132, + "theoretical_loss": 3.534123980024663, + "tokens_seen": 1415839744 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028840521564694083, + "loss": 2.5662, + "theoretical_loss": 3.5341093968609103, + "tokens_seen": 1415905280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028839518555667, + "loss": 2.736, + "theoretical_loss": 3.5340948145611195, + "tokens_seen": 1415970816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883851554663992, + "loss": 2.5976, + "theoretical_loss": 3.5340802331251995, + "tokens_seen": 1416036352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883751253761284, + "loss": 2.6624, + "theoretical_loss": 3.534065652553059, + "tokens_seen": 1416101888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883650952858576, + "loss": 2.7034, + "theoretical_loss": 3.5340510728446075, + "tokens_seen": 1416167424 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028835506519558674, + "loss": 2.7101, + "theoretical_loss": 3.5340364939997526, + "tokens_seen": 1416232960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028834503510531597, + "loss": 2.7455, + "theoretical_loss": 3.5340219160184043, + "tokens_seen": 1416298496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883350050150451, + "loss": 2.7348, + "theoretical_loss": 3.534007338900471, + "tokens_seen": 1416364032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028832497492477433, + "loss": 2.814, + "theoretical_loss": 3.5339927626458616, + "tokens_seen": 1416429568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883149448345035, + "loss": 2.5852, + "theoretical_loss": 3.533978187254485, + "tokens_seen": 1416495104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883049147442327, + "loss": 2.7235, + "theoretical_loss": 3.53396361272625, + "tokens_seen": 1416560640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882948846539619, + "loss": 2.6525, + "theoretical_loss": 3.533949039061067, + "tokens_seen": 1416626176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028828485456369106, + "loss": 2.761, + "theoretical_loss": 3.5339344662588426, + "tokens_seen": 1416691712 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882748244734203, + "loss": 2.5754, + "theoretical_loss": 3.533919894319488, + "tokens_seen": 1416757248 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882647943831495, + "loss": 2.5589, + "theoretical_loss": 3.5339053232429105, + "tokens_seen": 1416822784 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028825476429287866, + "loss": 2.4349, + "theoretical_loss": 3.53389075302902, + "tokens_seen": 1416888320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028824473420260784, + "loss": 2.5853, + "theoretical_loss": 3.533876183677725, + "tokens_seen": 1416953856 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288234704112337, + "loss": 2.5862, + "theoretical_loss": 3.5338616151889353, + "tokens_seen": 1417019392 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882246740220662, + "loss": 2.6607, + "theoretical_loss": 3.5338470475625594, + "tokens_seen": 1417084928 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028821464393179543, + "loss": 2.6889, + "theoretical_loss": 3.533832480798506, + "tokens_seen": 1417150464 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 781532, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.577233076095581, + "objective/train/theoretical_loss": 3.5338179148966855, + "objective/train/tokens_used": 1437676000, + "theoretical_loss": 3.5338179148966855, + "tokens_seen": 1417216000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028820461384152456, + "loss": 2.587, + "theoretical_loss": 3.5338179148966855, + "tokens_seen": 1417216000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881945837512538, + "loss": 2.6776, + "theoretical_loss": 3.533803349857006, + "tokens_seen": 1417281536 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288184553660983, + "loss": 2.6713, + "theoretical_loss": 3.533788785679376, + "tokens_seen": 1417347072 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028817452357071216, + "loss": 2.528, + "theoretical_loss": 3.5337742223637063, + "tokens_seen": 1417412608 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028816449348044134, + "loss": 2.6677, + "theoretical_loss": 3.533759659909905, + "tokens_seen": 1417478144 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881544633901705, + "loss": 2.7362, + "theoretical_loss": 3.5337450983178806, + "tokens_seen": 1417543680 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881444332998997, + "loss": 2.5738, + "theoretical_loss": 3.5337305375875436, + "tokens_seen": 1417609216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028813440320962894, + "loss": 2.6745, + "theoretical_loss": 3.5337159777188027, + "tokens_seen": 1417674752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028812437311935806, + "loss": 2.609, + "theoretical_loss": 3.5337014187115674, + "tokens_seen": 1417740288 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881143430290873, + "loss": 2.7586, + "theoretical_loss": 3.533686860565746, + "tokens_seen": 1417805824 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881043129388164, + "loss": 2.4825, + "theoretical_loss": 3.533672303281248, + "tokens_seen": 1417871360 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028809428284854566, + "loss": 2.7825, + "theoretical_loss": 3.5336577468579833, + "tokens_seen": 1417936896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028808425275827484, + "loss": 2.6336, + "theoretical_loss": 3.5336431912958606, + "tokens_seen": 1418002432 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288074222668004, + "loss": 2.5171, + "theoretical_loss": 3.5336286365947887, + "tokens_seen": 1418067968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880641925777332, + "loss": 2.836, + "theoretical_loss": 3.533614082754678, + "tokens_seen": 1418133504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880541624874624, + "loss": 2.7926, + "theoretical_loss": 3.5335995297754375, + "tokens_seen": 1418199040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028804413239719157, + "loss": 2.5955, + "theoretical_loss": 3.5335849776569757, + "tokens_seen": 1418264576 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880341023069208, + "loss": 2.6247, + "theoretical_loss": 3.5335704263992023, + "tokens_seen": 1418330112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028802407221664993, + "loss": 2.4715, + "theoretical_loss": 3.533555876002027, + "tokens_seen": 1418395648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028801404212637916, + "loss": 2.7533, + "theoretical_loss": 3.5335413264653592, + "tokens_seen": 1418461184 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028800401203610835, + "loss": 2.7117, + "theoretical_loss": 3.533526777789108, + "tokens_seen": 1418526720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879939819458375, + "loss": 2.84, + "theoretical_loss": 3.5335122299731827, + "tokens_seen": 1418592256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879839518555667, + "loss": 2.8294, + "theoretical_loss": 3.533497683017492, + "tokens_seen": 1418657792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879739217652959, + "loss": 2.2382, + "theoretical_loss": 3.533483136921947, + "tokens_seen": 1418723328 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028796389167502507, + "loss": 2.6418, + "theoretical_loss": 3.5334685916864554, + "tokens_seen": 1418788864 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 782242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.07771635055542, + "objective/train/theoretical_loss": 3.5334540473109275, + "objective/train/tokens_used": 1439314400, + "theoretical_loss": 3.5334540473109275, + "tokens_seen": 1418854400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879538615847543, + "loss": 2.4024, + "theoretical_loss": 3.5334540473109275, + "tokens_seen": 1418854400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028794383149448343, + "loss": 2.8049, + "theoretical_loss": 3.533439503795273, + "tokens_seen": 1418919936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028793380140421267, + "loss": 2.4388, + "theoretical_loss": 3.533424961139401, + "tokens_seen": 1418985472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879237713139418, + "loss": 2.6352, + "theoretical_loss": 3.5334104193432205, + "tokens_seen": 1419051008 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028791374122367103, + "loss": 2.5944, + "theoretical_loss": 3.5333958784066413, + "tokens_seen": 1419116544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879037111334002, + "loss": 2.7129, + "theoretical_loss": 3.5333813383295736, + "tokens_seen": 1419182080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878936810431294, + "loss": 2.4101, + "theoretical_loss": 3.533366799111926, + "tokens_seen": 1419247616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878836509528586, + "loss": 2.5793, + "theoretical_loss": 3.533352260753608, + "tokens_seen": 1419313152 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878736208625878, + "loss": 2.7171, + "theoretical_loss": 3.5333377232545295, + "tokens_seen": 1419378688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028786359077231694, + "loss": 2.8174, + "theoretical_loss": 3.5333231866146, + "tokens_seen": 1419444224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028785356068204617, + "loss": 2.4148, + "theoretical_loss": 3.5333086508337295, + "tokens_seen": 1419509760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878435305917753, + "loss": 2.7083, + "theoretical_loss": 3.5332941159118265, + "tokens_seen": 1419575296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028783350050150453, + "loss": 2.6961, + "theoretical_loss": 3.533279581848802, + "tokens_seen": 1419640832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878234704112337, + "loss": 2.4613, + "theoretical_loss": 3.533265048644564, + "tokens_seen": 1419706368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878134403209629, + "loss": 2.5528, + "theoretical_loss": 3.5332505162990238, + "tokens_seen": 1419771904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878034102306921, + "loss": 2.8151, + "theoretical_loss": 3.533235984812089, + "tokens_seen": 1419837440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028779338014042126, + "loss": 2.6704, + "theoretical_loss": 3.5332214541836713, + "tokens_seen": 1419902976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028778335005015044, + "loss": 2.6604, + "theoretical_loss": 3.5332069244136797, + "tokens_seen": 1419968512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877733199598797, + "loss": 2.5915, + "theoretical_loss": 3.533192395502023, + "tokens_seen": 1420034048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877632898696088, + "loss": 2.5496, + "theoretical_loss": 3.533177867448612, + "tokens_seen": 1420099584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028775325977933804, + "loss": 2.559, + "theoretical_loss": 3.5331633402533553, + "tokens_seen": 1420165120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028774322968906716, + "loss": 2.7361, + "theoretical_loss": 3.533148813916164, + "tokens_seen": 1420230656 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877331995987964, + "loss": 2.4624, + "theoretical_loss": 3.5331342884369468, + "tokens_seen": 1420296192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877231695085256, + "loss": 2.5075, + "theoretical_loss": 3.5331197638156135, + "tokens_seen": 1420361728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028771313941825476, + "loss": 2.5413, + "theoretical_loss": 3.533105240052074, + "tokens_seen": 1420427264 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 783364, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6914069652557373, + "objective/train/theoretical_loss": 3.533090717146238, + "objective/train/tokens_used": 1440952800, + "theoretical_loss": 3.533090717146238, + "tokens_seen": 1420492800 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028770310932798394, + "loss": 2.7826, + "theoretical_loss": 3.533090717146238, + "tokens_seen": 1420492800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876930792377132, + "loss": 2.7784, + "theoretical_loss": 3.5330761950980163, + "tokens_seen": 1420558336 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876830491474423, + "loss": 2.5332, + "theoretical_loss": 3.533061673907317, + "tokens_seen": 1420623872 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028767301905717154, + "loss": 2.5636, + "theoretical_loss": 3.533047153574051, + "tokens_seen": 1420689408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028766298896690067, + "loss": 2.6052, + "theoretical_loss": 3.5330326340981277, + "tokens_seen": 1420754944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876529588766299, + "loss": 2.4816, + "theoretical_loss": 3.5330181154794573, + "tokens_seen": 1420820480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876429287863591, + "loss": 2.6522, + "theoretical_loss": 3.533003597717949, + "tokens_seen": 1420886016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028763289869608826, + "loss": 2.7393, + "theoretical_loss": 3.5329890808135134, + "tokens_seen": 1420951552 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028762286860581745, + "loss": 2.4828, + "theoretical_loss": 3.53297456476606, + "tokens_seen": 1421017088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876128385155466, + "loss": 2.6096, + "theoretical_loss": 3.5329600495754985, + "tokens_seen": 1421082624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876028084252758, + "loss": 2.7357, + "theoretical_loss": 3.5329455352417396, + "tokens_seen": 1421148160 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028759277833500504, + "loss": 2.6544, + "theoretical_loss": 3.532931021764692, + "tokens_seen": 1421213696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028758274824473417, + "loss": 2.7388, + "theoretical_loss": 3.5329165091442665, + "tokens_seen": 1421279232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875727181544634, + "loss": 2.6667, + "theoretical_loss": 3.532901997380373, + "tokens_seen": 1421344768 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028756268806419253, + "loss": 2.8288, + "theoretical_loss": 3.532887486472921, + "tokens_seen": 1421410304 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028755265797392177, + "loss": 2.5947, + "theoretical_loss": 3.5328729764218214, + "tokens_seen": 1421475840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028754262788365095, + "loss": 2.6598, + "theoretical_loss": 3.5328584672269834, + "tokens_seen": 1421541376 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028753259779338013, + "loss": 2.3975, + "theoretical_loss": 3.532843958888317, + "tokens_seen": 1421606912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028752256770310936, + "loss": 2.5949, + "theoretical_loss": 3.532829451405732, + "tokens_seen": 1421672448 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028751253761283855, + "loss": 2.779, + "theoretical_loss": 3.532814944779139, + "tokens_seen": 1421737984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875025075225677, + "loss": 2.577, + "theoretical_loss": 3.532800439008448, + "tokens_seen": 1421803520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874924774322969, + "loss": 2.5758, + "theoretical_loss": 3.532785934093569, + "tokens_seen": 1421869056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874824473420261, + "loss": 2.6191, + "theoretical_loss": 3.5327714300344115, + "tokens_seen": 1421934592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028747241725175527, + "loss": 2.5578, + "theoretical_loss": 3.532756926830886, + "tokens_seen": 1422000128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874623871614845, + "loss": 2.5465, + "theoretical_loss": 3.532742424482903, + "tokens_seen": 1422065664 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 783958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4881510734558105, + "objective/train/theoretical_loss": 3.532727922990372, + "objective/train/tokens_used": 1442591200, + "theoretical_loss": 3.532727922990372, + "tokens_seen": 1422131200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028745235707121363, + "loss": 2.5889, + "theoretical_loss": 3.532727922990372, + "tokens_seen": 1422131200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028744232698094287, + "loss": 2.4254, + "theoretical_loss": 3.5327134223532033, + "tokens_seen": 1422196736 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287432296890672, + "loss": 2.7556, + "theoretical_loss": 3.5326989225713077, + "tokens_seen": 1422262272 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028742226680040123, + "loss": 2.8043, + "theoretical_loss": 3.532684423644594, + "tokens_seen": 1422327808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874122367101304, + "loss": 2.5499, + "theoretical_loss": 3.532669925572973, + "tokens_seen": 1422393344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874022066198596, + "loss": 2.7351, + "theoretical_loss": 3.532655428356356, + "tokens_seen": 1422458880 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873921765295888, + "loss": 2.5448, + "theoretical_loss": 3.532640931994651, + "tokens_seen": 1422524416 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287382146439318, + "loss": 2.6517, + "theoretical_loss": 3.53262643648777, + "tokens_seen": 1422589952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028737211634904714, + "loss": 2.7519, + "theoretical_loss": 3.5326119418356225, + "tokens_seen": 1422655488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028736208625877637, + "loss": 2.6739, + "theoretical_loss": 3.5325974480381186, + "tokens_seen": 1422721024 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873520561685055, + "loss": 2.8156, + "theoretical_loss": 3.5325829550951693, + "tokens_seen": 1422786560 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028734202607823473, + "loss": 2.4075, + "theoretical_loss": 3.532568463006684, + "tokens_seen": 1422852096 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873319959879639, + "loss": 2.6581, + "theoretical_loss": 3.532553971772573, + "tokens_seen": 1422917632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873219658976931, + "loss": 2.754, + "theoretical_loss": 3.532539481392747, + "tokens_seen": 1422983168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873119358074223, + "loss": 2.7735, + "theoretical_loss": 3.532524991867116, + "tokens_seen": 1423048704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028730190571715146, + "loss": 2.849, + "theoretical_loss": 3.5325105031955912, + "tokens_seen": 1423114240 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028729187562688064, + "loss": 2.5006, + "theoretical_loss": 3.532496015378081, + "tokens_seen": 1423179776 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872818455366099, + "loss": 2.671, + "theoretical_loss": 3.532481528414498, + "tokens_seen": 1423245312 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287271815446339, + "loss": 2.4799, + "theoretical_loss": 3.5324670423047513, + "tokens_seen": 1423310848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028726178535606824, + "loss": 2.6465, + "theoretical_loss": 3.5324525570487513, + "tokens_seen": 1423376384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028725175526579736, + "loss": 2.4529, + "theoretical_loss": 3.532438072646408, + "tokens_seen": 1423441920 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872417251755266, + "loss": 2.6795, + "theoretical_loss": 3.532423589097633, + "tokens_seen": 1423507456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872316950852558, + "loss": 2.7439, + "theoretical_loss": 3.5324091064023357, + "tokens_seen": 1423572992 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028722166499498496, + "loss": 2.4454, + "theoretical_loss": 3.532394624560427, + "tokens_seen": 1423638528 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028721163490471414, + "loss": 2.7103, + "theoretical_loss": 3.5323801435718174, + "tokens_seen": 1423704064 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 785018, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1949687004089355, + "objective/train/theoretical_loss": 3.5323656634364164, + "objective/train/tokens_used": 1444229600, + "theoretical_loss": 3.5323656634364164, + "tokens_seen": 1423769600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872016048144434, + "loss": 2.8826, + "theoretical_loss": 3.5323656634364164, + "tokens_seen": 1423769600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871915747241725, + "loss": 2.6976, + "theoretical_loss": 3.5323511841541357, + "tokens_seen": 1423835136 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028718154463390174, + "loss": 2.6035, + "theoretical_loss": 3.5323367057248856, + "tokens_seen": 1423900672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028717151454363087, + "loss": 2.6493, + "theoretical_loss": 3.5323222281485753, + "tokens_seen": 1423966208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871614844533601, + "loss": 2.6472, + "theoretical_loss": 3.532307751425117, + "tokens_seen": 1424031744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871514543630893, + "loss": 2.7908, + "theoretical_loss": 3.5322932755544203, + "tokens_seen": 1424097280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028714142427281846, + "loss": 2.5785, + "theoretical_loss": 3.532278800536396, + "tokens_seen": 1424162816 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028713139418254765, + "loss": 2.5572, + "theoretical_loss": 3.5322643263709543, + "tokens_seen": 1424228352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871213640922768, + "loss": 2.5964, + "theoretical_loss": 3.532249853058006, + "tokens_seen": 1424293888 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287111334002006, + "loss": 2.5513, + "theoretical_loss": 3.532235380597462, + "tokens_seen": 1424359424 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028710130391173524, + "loss": 2.6485, + "theoretical_loss": 3.532220908989232, + "tokens_seen": 1424424960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028709127382146437, + "loss": 2.9143, + "theoretical_loss": 3.5322064382332274, + "tokens_seen": 1424490496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870812437311936, + "loss": 2.5105, + "theoretical_loss": 3.5321919683293586, + "tokens_seen": 1424556032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028707121364092273, + "loss": 2.6386, + "theoretical_loss": 3.532177499277536, + "tokens_seen": 1424621568 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028706118355065197, + "loss": 2.6675, + "theoretical_loss": 3.5321630310776704, + "tokens_seen": 1424687104 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028705115346038115, + "loss": 2.587, + "theoretical_loss": 3.5321485637296726, + "tokens_seen": 1424752640 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028704112337011033, + "loss": 2.6668, + "theoretical_loss": 3.532134097233453, + "tokens_seen": 1424818176 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870310932798395, + "loss": 2.6154, + "theoretical_loss": 3.5321196315889223, + "tokens_seen": 1424883712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028702106318956875, + "loss": 2.6835, + "theoretical_loss": 3.5321051667959913, + "tokens_seen": 1424949248 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870110330992979, + "loss": 2.6394, + "theoretical_loss": 3.5320907028545707, + "tokens_seen": 1425014784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870010030090271, + "loss": 2.4343, + "theoretical_loss": 3.532076239764571, + "tokens_seen": 1425080320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028699097291875624, + "loss": 2.7033, + "theoretical_loss": 3.5320617775259033, + "tokens_seen": 1425145856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028698094282848547, + "loss": 2.6942, + "theoretical_loss": 3.5320473161384784, + "tokens_seen": 1425211392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028697091273821465, + "loss": 2.7405, + "theoretical_loss": 3.5320328556022065, + "tokens_seen": 1425276928 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028696088264794383, + "loss": 2.5886, + "theoretical_loss": 3.5320183959169986, + "tokens_seen": 1425342464 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 786235, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.591965913772583, + "objective/train/theoretical_loss": 3.532003937082766, + "objective/train/tokens_used": 1445868000, + "theoretical_loss": 3.532003937082766, + "tokens_seen": 1425408000 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286950852557673, + "loss": 2.6586, + "theoretical_loss": 3.532003937082766, + "tokens_seen": 1425408000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869408224674022, + "loss": 2.69, + "theoretical_loss": 3.531989479099419, + "tokens_seen": 1425473536 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869307923771314, + "loss": 2.6215, + "theoretical_loss": 3.531975021966868, + "tokens_seen": 1425539072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869207622868606, + "loss": 2.7199, + "theoretical_loss": 3.531960565685025, + "tokens_seen": 1425604608 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028691073219658974, + "loss": 2.5658, + "theoretical_loss": 3.5319461102537995, + "tokens_seen": 1425670144 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286900702106319, + "loss": 2.5154, + "theoretical_loss": 3.531931655673103, + "tokens_seen": 1425735680 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868906720160481, + "loss": 2.6294, + "theoretical_loss": 3.5319172019428464, + "tokens_seen": 1425801216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028688064192577734, + "loss": 2.7823, + "theoretical_loss": 3.531902749062941, + "tokens_seen": 1425866752 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868706118355065, + "loss": 2.538, + "theoretical_loss": 3.531888297033297, + "tokens_seen": 1425932288 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868605817452357, + "loss": 2.5781, + "theoretical_loss": 3.5318738458538252, + "tokens_seen": 1425997824 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868505516549649, + "loss": 2.664, + "theoretical_loss": 3.531859395524437, + "tokens_seen": 1426063360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868405215646941, + "loss": 2.8786, + "theoretical_loss": 3.5318449460450436, + "tokens_seen": 1426128896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028683049147442324, + "loss": 2.7226, + "theoretical_loss": 3.531830497415555, + "tokens_seen": 1426194432 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868204613841525, + "loss": 2.41, + "theoretical_loss": 3.531816049635883, + "tokens_seen": 1426259968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868104312938816, + "loss": 2.4417, + "theoretical_loss": 3.5318016027059382, + "tokens_seen": 1426325504 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028680040120361084, + "loss": 2.62, + "theoretical_loss": 3.531787156625631, + "tokens_seen": 1426391040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028679037111334, + "loss": 2.7438, + "theoretical_loss": 3.531772711394874, + "tokens_seen": 1426456576 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867803410230692, + "loss": 2.4956, + "theoretical_loss": 3.531758267013577, + "tokens_seen": 1426522112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028677031093279844, + "loss": 2.4738, + "theoretical_loss": 3.531743823481651, + "tokens_seen": 1426587648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028676028084252756, + "loss": 2.7312, + "theoretical_loss": 3.5317293807990073, + "tokens_seen": 1426653184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867502507522568, + "loss": 2.4711, + "theoretical_loss": 3.5317149389655573, + "tokens_seen": 1426718720 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286740220661986, + "loss": 2.7506, + "theoretical_loss": 3.5317004979812117, + "tokens_seen": 1426784256 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028673019057171516, + "loss": 2.5439, + "theoretical_loss": 3.531686057845881, + "tokens_seen": 1426849792 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028672016048144434, + "loss": 2.5135, + "theoretical_loss": 3.5316716185594776, + "tokens_seen": 1426915328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867101303911736, + "loss": 2.6132, + "theoretical_loss": 3.531657180121911, + "tokens_seen": 1426980864 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 786774, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.598083257675171, + "objective/train/theoretical_loss": 3.5316427425330943, + "objective/train/tokens_used": 1447506400, + "theoretical_loss": 3.5316427425330943, + "tokens_seen": 1427046400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867001003009027, + "loss": 2.6186, + "theoretical_loss": 3.5316427425330943, + "tokens_seen": 1427046400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028669007021063194, + "loss": 2.9038, + "theoretical_loss": 3.531628305792937, + "tokens_seen": 1427111936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028668004012036107, + "loss": 2.6803, + "theoretical_loss": 3.531613869901351, + "tokens_seen": 1427177472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866700100300903, + "loss": 2.6351, + "theoretical_loss": 3.531599434858247, + "tokens_seen": 1427243008 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866599799398195, + "loss": 2.5262, + "theoretical_loss": 3.5315850006635365, + "tokens_seen": 1427308544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028664994984954866, + "loss": 2.552, + "theoretical_loss": 3.531570567317131, + "tokens_seen": 1427374080 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028663991975927785, + "loss": 2.6389, + "theoretical_loss": 3.531556134818941, + "tokens_seen": 1427439616 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286629889669007, + "loss": 2.7612, + "theoretical_loss": 3.531541703168878, + "tokens_seen": 1427505152 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866198595787362, + "loss": 2.5592, + "theoretical_loss": 3.531527272366853, + "tokens_seen": 1427570688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028660982948846544, + "loss": 2.4948, + "theoretical_loss": 3.531512842412778, + "tokens_seen": 1427636224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028659979939819457, + "loss": 2.6488, + "theoretical_loss": 3.531498413306564, + "tokens_seen": 1427701760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865897693079238, + "loss": 2.7581, + "theoretical_loss": 3.5314839850481214, + "tokens_seen": 1427767296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028657973921765293, + "loss": 2.6461, + "theoretical_loss": 3.5314695576373625, + "tokens_seen": 1427832832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028656970912738217, + "loss": 2.6874, + "theoretical_loss": 3.531455131074198, + "tokens_seen": 1427898368 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028655967903711135, + "loss": 2.625, + "theoretical_loss": 3.531440705358539, + "tokens_seen": 1427963904 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028654964894684053, + "loss": 2.5656, + "theoretical_loss": 3.5314262804902974, + "tokens_seen": 1428029440 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865396188565697, + "loss": 2.5828, + "theoretical_loss": 3.5314118564693846, + "tokens_seen": 1428094976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028652958876629895, + "loss": 2.3775, + "theoretical_loss": 3.531397433295712, + "tokens_seen": 1428160512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865195586760281, + "loss": 2.9139, + "theoretical_loss": 3.53138301096919, + "tokens_seen": 1428226048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865095285857573, + "loss": 2.5729, + "theoretical_loss": 3.5313685894897313, + "tokens_seen": 1428291584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028649949849548644, + "loss": 2.593, + "theoretical_loss": 3.531354168857246, + "tokens_seen": 1428357120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028648946840521567, + "loss": 2.5411, + "theoretical_loss": 3.5313397490716465, + "tokens_seen": 1428422656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028647943831494485, + "loss": 2.6507, + "theoretical_loss": 3.5313253301328436, + "tokens_seen": 1428488192 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028646940822467403, + "loss": 2.4573, + "theoretical_loss": 3.531310912040749, + "tokens_seen": 1428553728 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864593781344032, + "loss": 2.5802, + "theoretical_loss": 3.5312964947952743, + "tokens_seen": 1428619264 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 787815, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.069486379623413, + "objective/train/theoretical_loss": 3.5312820783963303, + "objective/train/tokens_used": 1449144800, + "theoretical_loss": 3.5312820783963303, + "tokens_seen": 1428684800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864493480441324, + "loss": 2.5377, + "theoretical_loss": 3.5312820783963303, + "tokens_seen": 1428684800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864393179538616, + "loss": 2.7072, + "theoretical_loss": 3.5312676628438293, + "tokens_seen": 1428750336 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864292878635908, + "loss": 2.567, + "theoretical_loss": 3.531253248137682, + "tokens_seen": 1428815872 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028641925777331994, + "loss": 2.8516, + "theoretical_loss": 3.5312388342778007, + "tokens_seen": 1428881408 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864092276830492, + "loss": 2.6305, + "theoretical_loss": 3.5312244212640964, + "tokens_seen": 1428946944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863991975927783, + "loss": 2.917, + "theoretical_loss": 3.5312100090964806, + "tokens_seen": 1429012480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028638916750250754, + "loss": 2.5045, + "theoretical_loss": 3.531195597774865, + "tokens_seen": 1429078016 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863791374122367, + "loss": 2.7884, + "theoretical_loss": 3.531181187299161, + "tokens_seen": 1429143552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863691073219659, + "loss": 2.744, + "theoretical_loss": 3.5311667776692803, + "tokens_seen": 1429209088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863590772316951, + "loss": 2.3637, + "theoretical_loss": 3.5311523688851345, + "tokens_seen": 1429274624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863490471414243, + "loss": 2.8131, + "theoretical_loss": 3.5311379609466353, + "tokens_seen": 1429340160 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028633901705115344, + "loss": 2.5255, + "theoretical_loss": 3.5311235538536936, + "tokens_seen": 1429405696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863289869608827, + "loss": 2.5705, + "theoretical_loss": 3.5311091476062217, + "tokens_seen": 1429471232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863189568706118, + "loss": 2.6134, + "theoretical_loss": 3.5310947422041314, + "tokens_seen": 1429536768 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028630892678034104, + "loss": 2.4821, + "theoretical_loss": 3.5310803376473334, + "tokens_seen": 1429602304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862988966900702, + "loss": 2.5878, + "theoretical_loss": 3.53106593393574, + "tokens_seen": 1429667840 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862888665997994, + "loss": 2.5424, + "theoretical_loss": 3.531051531069263, + "tokens_seen": 1429733376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862788365095286, + "loss": 2.6448, + "theoretical_loss": 3.5310371290478146, + "tokens_seen": 1429798912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028626880641925776, + "loss": 2.6663, + "theoretical_loss": 3.5310227278713047, + "tokens_seen": 1429864448 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028625877632898694, + "loss": 2.5672, + "theoretical_loss": 3.5310083275396464, + "tokens_seen": 1429929984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862487462387162, + "loss": 2.7178, + "theoretical_loss": 3.5309939280527516, + "tokens_seen": 1429995520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862387161484453, + "loss": 2.6479, + "theoretical_loss": 3.530979529410531, + "tokens_seen": 1430061056 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028622868605817454, + "loss": 2.7163, + "theoretical_loss": 3.5309651316128967, + "tokens_seen": 1430126592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028621865596790367, + "loss": 2.6371, + "theoretical_loss": 3.5309507346597613, + "tokens_seen": 1430192128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862086258776329, + "loss": 2.7673, + "theoretical_loss": 3.5309363385510353, + "tokens_seen": 1430257664 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 788429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.007455587387085, + "objective/train/theoretical_loss": 3.5309219432866312, + "objective/train/tokens_used": 1450783200, + "theoretical_loss": 3.5309219432866312, + "tokens_seen": 1430323200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861985957873621, + "loss": 2.8257, + "theoretical_loss": 3.5309219432866312, + "tokens_seen": 1430323200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028618856569709127, + "loss": 2.56, + "theoretical_loss": 3.5309075488664607, + "tokens_seen": 1430388736 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028617853560682045, + "loss": 2.7202, + "theoretical_loss": 3.530893155290436, + "tokens_seen": 1430454272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861685055165497, + "loss": 2.4225, + "theoretical_loss": 3.5308787625584683, + "tokens_seen": 1430519808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861584754262788, + "loss": 2.4291, + "theoretical_loss": 3.5308643706704697, + "tokens_seen": 1430585344 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028614844533600805, + "loss": 2.6722, + "theoretical_loss": 3.5308499796263515, + "tokens_seen": 1430650880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028613841524573717, + "loss": 2.7396, + "theoretical_loss": 3.530835589426027, + "tokens_seen": 1430716416 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861283851554664, + "loss": 2.7058, + "theoretical_loss": 3.530821200069407, + "tokens_seen": 1430781952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861183550651956, + "loss": 2.6886, + "theoretical_loss": 3.5308068115564026, + "tokens_seen": 1430847488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028610832497492477, + "loss": 2.5916, + "theoretical_loss": 3.5307924238869273, + "tokens_seen": 1430913024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028609829488465395, + "loss": 2.5547, + "theoretical_loss": 3.5307780370608923, + "tokens_seen": 1430978560 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028608826479438313, + "loss": 2.4528, + "theoretical_loss": 3.53076365107821, + "tokens_seen": 1431044096 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860782347041123, + "loss": 2.6304, + "theoretical_loss": 3.530749265938791, + "tokens_seen": 1431109632 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028606820461384155, + "loss": 2.5961, + "theoretical_loss": 3.530734881642549, + "tokens_seen": 1431175168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860581745235707, + "loss": 2.6846, + "theoretical_loss": 3.530720498189395, + "tokens_seen": 1431240704 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860481444332999, + "loss": 2.6643, + "theoretical_loss": 3.5307061155792416, + "tokens_seen": 1431306240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860381143430291, + "loss": 2.6204, + "theoretical_loss": 3.5306917338119996, + "tokens_seen": 1431371776 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860280842527583, + "loss": 2.6394, + "theoretical_loss": 3.530677352887582, + "tokens_seen": 1431437312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860180541624875, + "loss": 2.6282, + "theoretical_loss": 3.530662972805901, + "tokens_seen": 1431502848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028600802407221664, + "loss": 2.8499, + "theoretical_loss": 3.530648593566868, + "tokens_seen": 1431568384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028599799398194587, + "loss": 2.5693, + "theoretical_loss": 3.5306342151703953, + "tokens_seen": 1431633920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028598796389167505, + "loss": 2.7649, + "theoretical_loss": 3.5306198376163946, + "tokens_seen": 1431699456 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028597793380140423, + "loss": 2.7873, + "theoretical_loss": 3.5306054609047783, + "tokens_seen": 1431764992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859679037111334, + "loss": 2.676, + "theoretical_loss": 3.530591085035459, + "tokens_seen": 1431830528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859578736208626, + "loss": 2.7262, + "theoretical_loss": 3.530576710008348, + "tokens_seen": 1431896064 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 789812, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3145503997802734, + "objective/train/theoretical_loss": 3.5305623358233578, + "objective/train/tokens_used": 1452421600, + "theoretical_loss": 3.5305623358233578, + "tokens_seen": 1431961600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859478435305918, + "loss": 2.6037, + "theoretical_loss": 3.5305623358233578, + "tokens_seen": 1431961600 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285937813440321, + "loss": 2.6274, + "theoretical_loss": 3.5305479624804006, + "tokens_seen": 1432027136 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028592778335005014, + "loss": 2.543, + "theoretical_loss": 3.530533589979388, + "tokens_seen": 1432092672 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859177532597794, + "loss": 2.5528, + "theoretical_loss": 3.530519218320233, + "tokens_seen": 1432158208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859077231695085, + "loss": 2.5332, + "theoretical_loss": 3.530504847502847, + "tokens_seen": 1432223744 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028589769307923774, + "loss": 2.7996, + "theoretical_loss": 3.5304904775271426, + "tokens_seen": 1432289280 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858876629889669, + "loss": 2.495, + "theoretical_loss": 3.530476108393032, + "tokens_seen": 1432354816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858776328986961, + "loss": 2.5663, + "theoretical_loss": 3.530461740100427, + "tokens_seen": 1432420352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858676028084253, + "loss": 2.899, + "theoretical_loss": 3.5304473726492405, + "tokens_seen": 1432485888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858575727181545, + "loss": 2.7768, + "theoretical_loss": 3.5304330060393845, + "tokens_seen": 1432551424 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028584754262788364, + "loss": 2.6372, + "theoretical_loss": 3.5304186402707707, + "tokens_seen": 1432616960 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858375125376129, + "loss": 2.7577, + "theoretical_loss": 3.5304042753433116, + "tokens_seen": 1432682496 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285827482447342, + "loss": 2.6304, + "theoretical_loss": 3.53038991125692, + "tokens_seen": 1432748032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028581745235707124, + "loss": 2.7153, + "theoretical_loss": 3.530375548011508, + "tokens_seen": 1432813568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858074222668004, + "loss": 2.695, + "theoretical_loss": 3.5303611856069876, + "tokens_seen": 1432879104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857973921765296, + "loss": 2.3783, + "theoretical_loss": 3.5303468240432707, + "tokens_seen": 1432944640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857873620862588, + "loss": 2.8994, + "theoretical_loss": 3.5303324633202706, + "tokens_seen": 1433010176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028577733199598796, + "loss": 2.5932, + "theoretical_loss": 3.5303181034378994, + "tokens_seen": 1433075712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028576730190571714, + "loss": 2.5311, + "theoretical_loss": 3.530303744396069, + "tokens_seen": 1433141248 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857572718154464, + "loss": 2.5577, + "theoretical_loss": 3.5302893861946925, + "tokens_seen": 1433206784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857472417251755, + "loss": 2.4627, + "theoretical_loss": 3.5302750288336817, + "tokens_seen": 1433272320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028573721163490474, + "loss": 2.7041, + "theoretical_loss": 3.5302606723129486, + "tokens_seen": 1433337856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028572718154463387, + "loss": 2.6183, + "theoretical_loss": 3.5302463166324065, + "tokens_seen": 1433403392 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857171514543631, + "loss": 2.8183, + "theoretical_loss": 3.5302319617919675, + "tokens_seen": 1433468928 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857071213640923, + "loss": 2.701, + "theoretical_loss": 3.5302176077915437, + "tokens_seen": 1433534464 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 791077, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5977752208709717, + "objective/train/theoretical_loss": 3.530203254631048, + "objective/train/tokens_used": 1454060000, + "theoretical_loss": 3.530203254631048, + "tokens_seen": 1433600000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028569709127382147, + "loss": 2.5736, + "theoretical_loss": 3.530203254631048, + "tokens_seen": 1433600000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028568706118355065, + "loss": 2.733, + "theoretical_loss": 3.5301889023103925, + "tokens_seen": 1433665536 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856770310932799, + "loss": 2.4584, + "theoretical_loss": 3.5301745508294893, + "tokens_seen": 1433731072 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285667001003009, + "loss": 2.4638, + "theoretical_loss": 3.530160200188252, + "tokens_seen": 1433796608 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028565697091273825, + "loss": 2.8465, + "theoretical_loss": 3.530145850386593, + "tokens_seen": 1433862144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028564694082246737, + "loss": 2.5399, + "theoretical_loss": 3.5301315014244237, + "tokens_seen": 1433927680 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856369107321966, + "loss": 2.4459, + "theoretical_loss": 3.5301171533016573, + "tokens_seen": 1433993216 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856268806419258, + "loss": 2.2933, + "theoretical_loss": 3.5301028060182063, + "tokens_seen": 1434058752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028561685055165497, + "loss": 2.5866, + "theoretical_loss": 3.5300884595739834, + "tokens_seen": 1434124288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028560682046138415, + "loss": 2.7661, + "theoretical_loss": 3.530074113968901, + "tokens_seen": 1434189824 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028559679037111333, + "loss": 2.8626, + "theoretical_loss": 3.5300597692028717, + "tokens_seen": 1434255360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855867602808425, + "loss": 2.5412, + "theoretical_loss": 3.530045425275808, + "tokens_seen": 1434320896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028557673019057175, + "loss": 2.579, + "theoretical_loss": 3.530031082187622, + "tokens_seen": 1434386432 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855667001003009, + "loss": 2.5718, + "theoretical_loss": 3.5300167399382274, + "tokens_seen": 1434451968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855566700100301, + "loss": 2.7666, + "theoretical_loss": 3.5300023985275364, + "tokens_seen": 1434517504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855466399197593, + "loss": 2.5841, + "theoretical_loss": 3.5299880579554612, + "tokens_seen": 1434583040 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855366098294885, + "loss": 2.5258, + "theoretical_loss": 3.5299737182219153, + "tokens_seen": 1434648576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028552657973921765, + "loss": 2.6316, + "theoretical_loss": 3.5299593793268107, + "tokens_seen": 1434714112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028551654964894684, + "loss": 2.5786, + "theoretical_loss": 3.52994504127006, + "tokens_seen": 1434779648 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285506519558676, + "loss": 2.968, + "theoretical_loss": 3.5299307040515764, + "tokens_seen": 1434845184 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028549648946840525, + "loss": 2.5524, + "theoretical_loss": 3.529916367671272, + "tokens_seen": 1434910720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854864593781344, + "loss": 2.6716, + "theoretical_loss": 3.5299020321290606, + "tokens_seen": 1434976256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854764292878636, + "loss": 2.7411, + "theoretical_loss": 3.529887697424854, + "tokens_seen": 1435041792 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028546639919759274, + "loss": 2.8467, + "theoretical_loss": 3.529873363558565, + "tokens_seen": 1435107328 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285456369107322, + "loss": 2.5434, + "theoretical_loss": 3.529859030530106, + "tokens_seen": 1435172864 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 791531, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.479179620742798, + "objective/train/theoretical_loss": 3.529844698339391, + "objective/train/tokens_used": 1455698400, + "theoretical_loss": 3.529844698339391, + "tokens_seen": 1435238400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028544633901705116, + "loss": 2.6274, + "theoretical_loss": 3.529844698339391, + "tokens_seen": 1435238400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028543630892678034, + "loss": 2.9353, + "theoretical_loss": 3.5298303669863316, + "tokens_seen": 1435303936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854262788365095, + "loss": 2.455, + "theoretical_loss": 3.5298160364708413, + "tokens_seen": 1435369472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854162487462387, + "loss": 2.7144, + "theoretical_loss": 3.5298017067928327, + "tokens_seen": 1435435008 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002854062186559679, + "loss": 2.6812, + "theoretical_loss": 3.5297873779522186, + "tokens_seen": 1435500544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853961885656971, + "loss": 2.6906, + "theoretical_loss": 3.529773049948912, + "tokens_seen": 1435566080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028538615847542624, + "loss": 2.6389, + "theoretical_loss": 3.5297587227828253, + "tokens_seen": 1435631616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853761283851555, + "loss": 2.5088, + "theoretical_loss": 3.529744396453872, + "tokens_seen": 1435697152 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028536609829488466, + "loss": 2.6032, + "theoretical_loss": 3.529730070961964, + "tokens_seen": 1435762688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028535606820461384, + "loss": 2.5452, + "theoretical_loss": 3.5297157463070157, + "tokens_seen": 1435828224 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285346038114343, + "loss": 2.6159, + "theoretical_loss": 3.5297014224889383, + "tokens_seen": 1435893760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853360080240722, + "loss": 2.5628, + "theoretical_loss": 3.529687099507646, + "tokens_seen": 1435959296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853259779338014, + "loss": 2.4744, + "theoretical_loss": 3.5296727773630514, + "tokens_seen": 1436024832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853159478435306, + "loss": 2.7097, + "theoretical_loss": 3.529658456055067, + "tokens_seen": 1436090368 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028530591775325975, + "loss": 2.7462, + "theoretical_loss": 3.5296441355836063, + "tokens_seen": 1436155904 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285295887662989, + "loss": 2.6551, + "theoretical_loss": 3.529629815948582, + "tokens_seen": 1436221440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852858575727181, + "loss": 2.8108, + "theoretical_loss": 3.5296154971499067, + "tokens_seen": 1436286976 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028527582748244735, + "loss": 2.8623, + "theoretical_loss": 3.5296011791874946, + "tokens_seen": 1436352512 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852657973921766, + "loss": 2.7176, + "theoretical_loss": 3.5295868620612576, + "tokens_seen": 1436418048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852557673019057, + "loss": 2.6384, + "theoretical_loss": 3.5295725457711087, + "tokens_seen": 1436483584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028524573721163494, + "loss": 2.6418, + "theoretical_loss": 3.5295582303169617, + "tokens_seen": 1436549120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028523570712136407, + "loss": 2.536, + "theoretical_loss": 3.5295439156987287, + "tokens_seen": 1436614656 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852256770310933, + "loss": 2.5104, + "theoretical_loss": 3.529529601916324, + "tokens_seen": 1436680192 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852156469408225, + "loss": 2.7834, + "theoretical_loss": 3.5295152889696597, + "tokens_seen": 1436745728 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028520561685055167, + "loss": 2.7306, + "theoretical_loss": 3.5295009768586487, + "tokens_seen": 1436811264 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 792701, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7126412391662598, + "objective/train/theoretical_loss": 3.529486665583205, + "objective/train/tokens_used": 1457336800, + "theoretical_loss": 3.529486665583205, + "tokens_seen": 1436876800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028519558676028085, + "loss": 2.6347, + "theoretical_loss": 3.529486665583205, + "tokens_seen": 1436876800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851855566700101, + "loss": 2.7781, + "theoretical_loss": 3.529472355143241, + "tokens_seen": 1436942336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851755265797392, + "loss": 2.7436, + "theoretical_loss": 3.52945804553867, + "tokens_seen": 1437007872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028516549648946845, + "loss": 2.6782, + "theoretical_loss": 3.5294437367694056, + "tokens_seen": 1437073408 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028515546639919757, + "loss": 2.6728, + "theoretical_loss": 3.5294294288353605, + "tokens_seen": 1437138944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851454363089268, + "loss": 2.7392, + "theoretical_loss": 3.529415121736448, + "tokens_seen": 1437204480 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285135406218656, + "loss": 2.5219, + "theoretical_loss": 3.529400815472581, + "tokens_seen": 1437270016 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028512537612838517, + "loss": 2.7083, + "theoretical_loss": 3.5293865100436728, + "tokens_seen": 1437335552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028511534603811435, + "loss": 2.7232, + "theoretical_loss": 3.5293722054496364, + "tokens_seen": 1437401088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028510531594784353, + "loss": 2.8735, + "theoretical_loss": 3.5293579016903855, + "tokens_seen": 1437466624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850952858575727, + "loss": 2.6501, + "theoretical_loss": 3.5293435987658333, + "tokens_seen": 1437532160 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028508525576730195, + "loss": 2.5175, + "theoretical_loss": 3.529329296675893, + "tokens_seen": 1437597696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850752256770311, + "loss": 2.8487, + "theoretical_loss": 3.5293149954204774, + "tokens_seen": 1437663232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850651955867603, + "loss": 2.4862, + "theoretical_loss": 3.5293006949995007, + "tokens_seen": 1437728768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850551654964895, + "loss": 2.5771, + "theoretical_loss": 3.529286395412875, + "tokens_seen": 1437794304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850451354062187, + "loss": 2.6454, + "theoretical_loss": 3.5292720966605144, + "tokens_seen": 1437859840 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028503510531594785, + "loss": 2.5789, + "theoretical_loss": 3.5292577987423317, + "tokens_seen": 1437925376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028502507522567704, + "loss": 2.6764, + "theoretical_loss": 3.529243501658241, + "tokens_seen": 1437990912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850150451354062, + "loss": 2.618, + "theoretical_loss": 3.5292292054081544, + "tokens_seen": 1438056448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028500501504513545, + "loss": 2.5581, + "theoretical_loss": 3.5292149099919867, + "tokens_seen": 1438121984 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849949849548646, + "loss": 2.7268, + "theoretical_loss": 3.52920061540965, + "tokens_seen": 1438187520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849849548645938, + "loss": 2.6949, + "theoretical_loss": 3.529186321661059, + "tokens_seen": 1438253056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028497492477432294, + "loss": 2.6476, + "theoretical_loss": 3.5291720287461255, + "tokens_seen": 1438318592 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849648946840522, + "loss": 2.4817, + "theoretical_loss": 3.5291577366647635, + "tokens_seen": 1438384128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028495486459378136, + "loss": 2.4231, + "theoretical_loss": 3.529143445416887, + "tokens_seen": 1438449664 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 793361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5532398223876953, + "objective/train/theoretical_loss": 3.5291291550024093, + "objective/train/tokens_used": 1458975200, + "theoretical_loss": 3.5291291550024093, + "tokens_seen": 1438515200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028494483450351054, + "loss": 2.614, + "theoretical_loss": 3.5291291550024093, + "tokens_seen": 1438515200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849348044132397, + "loss": 2.7064, + "theoretical_loss": 3.529114865421243, + "tokens_seen": 1438580736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849247743229689, + "loss": 2.5424, + "theoretical_loss": 3.5291005766733026, + "tokens_seen": 1438646272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849147442326981, + "loss": 2.8834, + "theoretical_loss": 3.5290862887585, + "tokens_seen": 1438711808 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849047141424273, + "loss": 2.808, + "theoretical_loss": 3.5290720016767514, + "tokens_seen": 1438777344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028489468405215644, + "loss": 2.4885, + "theoretical_loss": 3.5290577154279674, + "tokens_seen": 1438842880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848846539618857, + "loss": 2.9221, + "theoretical_loss": 3.529043430012063, + "tokens_seen": 1438908416 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028487462387161486, + "loss": 2.5549, + "theoretical_loss": 3.5290291454289515, + "tokens_seen": 1438973952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028486459378134404, + "loss": 2.589, + "theoretical_loss": 3.5290148616785464, + "tokens_seen": 1439039488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848545636910732, + "loss": 2.6573, + "theoretical_loss": 3.529000578760761, + "tokens_seen": 1439105024 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848445336008024, + "loss": 2.6267, + "theoretical_loss": 3.5289862966755092, + "tokens_seen": 1439170560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848345035105316, + "loss": 2.6466, + "theoretical_loss": 3.5289720154227044, + "tokens_seen": 1439236096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848244734202608, + "loss": 2.5453, + "theoretical_loss": 3.52895773500226, + "tokens_seen": 1439301632 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028481444332998995, + "loss": 2.4943, + "theoretical_loss": 3.52894345541409, + "tokens_seen": 1439367168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848044132397192, + "loss": 2.7117, + "theoretical_loss": 3.528929176658108, + "tokens_seen": 1439432704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847943831494483, + "loss": 2.6462, + "theoretical_loss": 3.528914898734227, + "tokens_seen": 1439498240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028478435305917755, + "loss": 2.4815, + "theoretical_loss": 3.528900621642361, + "tokens_seen": 1439563776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847743229689067, + "loss": 2.6197, + "theoretical_loss": 3.528886345382424, + "tokens_seen": 1439629312 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847642928786359, + "loss": 2.8349, + "theoretical_loss": 3.5288720699543292, + "tokens_seen": 1439694848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847542627883651, + "loss": 2.7815, + "theoretical_loss": 3.52885779535799, + "tokens_seen": 1439760384 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028474423269809427, + "loss": 2.5242, + "theoretical_loss": 3.5288435215933207, + "tokens_seen": 1439825920 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028473420260782345, + "loss": 2.5404, + "theoretical_loss": 3.5288292486602355, + "tokens_seen": 1439891456 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847241725175527, + "loss": 2.6967, + "theoretical_loss": 3.5288149765586465, + "tokens_seen": 1439956992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847141424272818, + "loss": 2.6013, + "theoretical_loss": 3.5288007052884685, + "tokens_seen": 1440022528 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028470411233701105, + "loss": 2.5721, + "theoretical_loss": 3.528786434849615, + "tokens_seen": 1440088064 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 794550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.535278558731079, + "objective/train/theoretical_loss": 3.5287721652419997, + "objective/train/tokens_used": 1460613600, + "theoretical_loss": 3.5287721652419997, + "tokens_seen": 1440153600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028469408224674023, + "loss": 2.6271, + "theoretical_loss": 3.5287721652419997, + "tokens_seen": 1440153600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846840521564694, + "loss": 2.4973, + "theoretical_loss": 3.528757896465537, + "tokens_seen": 1440219136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846740220661986, + "loss": 2.6563, + "theoretical_loss": 3.5287436285201395, + "tokens_seen": 1440284672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846639919759278, + "loss": 2.7874, + "theoretical_loss": 3.5287293614057216, + "tokens_seen": 1440350208 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028465396188565695, + "loss": 2.5078, + "theoretical_loss": 3.5287150951221973, + "tokens_seen": 1440415744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846439317953862, + "loss": 2.3781, + "theoretical_loss": 3.5287008296694804, + "tokens_seen": 1440481280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846339017051153, + "loss": 2.5335, + "theoretical_loss": 3.528686565047484, + "tokens_seen": 1440546816 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028462387161484455, + "loss": 2.5804, + "theoretical_loss": 3.5286723012561225, + "tokens_seen": 1440612352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846138415245737, + "loss": 2.6424, + "theoretical_loss": 3.52865803829531, + "tokens_seen": 1440677888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846038114343029, + "loss": 2.6213, + "theoretical_loss": 3.52864377616496, + "tokens_seen": 1440743424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845937813440321, + "loss": 2.658, + "theoretical_loss": 3.5286295148649867, + "tokens_seen": 1440808960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845837512537613, + "loss": 2.7613, + "theoretical_loss": 3.5286152543953033, + "tokens_seen": 1440874496 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028457372116349046, + "loss": 2.6722, + "theoretical_loss": 3.528600994755824, + "tokens_seen": 1440940032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845636910732197, + "loss": 2.6129, + "theoretical_loss": 3.528586735946463, + "tokens_seen": 1441005568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845536609829488, + "loss": 2.4905, + "theoretical_loss": 3.528572477967134, + "tokens_seen": 1441071104 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028454363089267805, + "loss": 2.8319, + "theoretical_loss": 3.528558220817751, + "tokens_seen": 1441136640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028453360080240724, + "loss": 2.8369, + "theoretical_loss": 3.528543964498228, + "tokens_seen": 1441202176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845235707121364, + "loss": 2.5964, + "theoretical_loss": 3.528529709008479, + "tokens_seen": 1441267712 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028451354062186565, + "loss": 2.6241, + "theoretical_loss": 3.5285154543484176, + "tokens_seen": 1441333248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845035105315948, + "loss": 2.6402, + "theoretical_loss": 3.528501200517958, + "tokens_seen": 1441398784 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284493480441324, + "loss": 2.5355, + "theoretical_loss": 3.528486947517015, + "tokens_seen": 1441464320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028448345035105314, + "loss": 2.5674, + "theoretical_loss": 3.528472695345501, + "tokens_seen": 1441529856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844734202607824, + "loss": 2.7757, + "theoretical_loss": 3.5284584440033315, + "tokens_seen": 1441595392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028446339017051156, + "loss": 2.6715, + "theoretical_loss": 3.5284441934904196, + "tokens_seen": 1441660928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028445336008024074, + "loss": 2.6472, + "theoretical_loss": 3.5284299438066804, + "tokens_seen": 1441726464 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 795512, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.223959445953369, + "objective/train/theoretical_loss": 3.5284156949520264, + "objective/train/tokens_used": 1462252000, + "theoretical_loss": 3.5284156949520264, + "tokens_seen": 1441792000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844433299899699, + "loss": 2.4238, + "theoretical_loss": 3.5284156949520264, + "tokens_seen": 1441792000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844332998996991, + "loss": 2.8375, + "theoretical_loss": 3.528401446926373, + "tokens_seen": 1441857536 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844232698094283, + "loss": 2.7861, + "theoretical_loss": 3.528387199729634, + "tokens_seen": 1441923072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844132397191575, + "loss": 2.5875, + "theoretical_loss": 3.528372953361723, + "tokens_seen": 1441988608 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028440320962888664, + "loss": 2.65, + "theoretical_loss": 3.5283587078225542, + "tokens_seen": 1442054144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843931795386159, + "loss": 2.6652, + "theoretical_loss": 3.5283444631120426, + "tokens_seen": 1442119680 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028438314944834506, + "loss": 2.7144, + "theoretical_loss": 3.528330219230102, + "tokens_seen": 1442185216 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028437311935807424, + "loss": 2.7403, + "theoretical_loss": 3.5283159761766454, + "tokens_seen": 1442250752 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843630892678034, + "loss": 2.6111, + "theoretical_loss": 3.5283017339515883, + "tokens_seen": 1442316288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843530591775326, + "loss": 2.6709, + "theoretical_loss": 3.5282874925548446, + "tokens_seen": 1442381824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843430290872618, + "loss": 2.5262, + "theoretical_loss": 3.5282732519863282, + "tokens_seen": 1442447360 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284332998996991, + "loss": 2.6623, + "theoretical_loss": 3.5282590122459534, + "tokens_seen": 1442512896 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028432296890672015, + "loss": 2.6434, + "theoretical_loss": 3.5282447733336344, + "tokens_seen": 1442578432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843129388164494, + "loss": 2.6882, + "theoretical_loss": 3.5282305352492855, + "tokens_seen": 1442643968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843029087261785, + "loss": 2.6735, + "theoretical_loss": 3.528216297992821, + "tokens_seen": 1442709504 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028429287863590775, + "loss": 2.7104, + "theoretical_loss": 3.5282020615641554, + "tokens_seen": 1442775040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842828485456369, + "loss": 2.7093, + "theoretical_loss": 3.5281878259632027, + "tokens_seen": 1442840576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842728184553661, + "loss": 2.6069, + "theoretical_loss": 3.5281735911898764, + "tokens_seen": 1442906112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842627883650953, + "loss": 2.7555, + "theoretical_loss": 3.5281593572440926, + "tokens_seen": 1442971648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028425275827482447, + "loss": 2.5602, + "theoretical_loss": 3.5281451241257638, + "tokens_seen": 1443037184 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028424272818455365, + "loss": 2.8941, + "theoretical_loss": 3.528130891834805, + "tokens_seen": 1443102720 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842326980942829, + "loss": 2.6407, + "theoretical_loss": 3.528116660371131, + "tokens_seen": 1443168256 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284222668004012, + "loss": 2.8021, + "theoretical_loss": 3.528102429734656, + "tokens_seen": 1443233792 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028421263791374125, + "loss": 2.4425, + "theoretical_loss": 3.5280881999252935, + "tokens_seen": 1443299328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028420260782347043, + "loss": 2.434, + "theoretical_loss": 3.528073970942959, + "tokens_seen": 1443364864 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 796104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4091320037841797, + "objective/train/theoretical_loss": 3.5280597427875664, + "objective/train/tokens_used": 1463890400, + "theoretical_loss": 3.5280597427875664, + "tokens_seen": 1443430400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841925777331996, + "loss": 2.4999, + "theoretical_loss": 3.5280597427875664, + "tokens_seen": 1443430400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841825476429288, + "loss": 2.7, + "theoretical_loss": 3.52804551545903, + "tokens_seen": 1443495936 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284172517552658, + "loss": 2.5288, + "theoretical_loss": 3.528031288957264, + "tokens_seen": 1443561472 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028416248746238715, + "loss": 2.6487, + "theoretical_loss": 3.5280170632821832, + "tokens_seen": 1443627008 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841524573721164, + "loss": 2.8229, + "theoretical_loss": 3.528002838433702, + "tokens_seen": 1443692544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841424272818455, + "loss": 2.8376, + "theoretical_loss": 3.527988614411735, + "tokens_seen": 1443758080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028413239719157475, + "loss": 2.4499, + "theoretical_loss": 3.5279743912161963, + "tokens_seen": 1443823616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841223671013039, + "loss": 2.6224, + "theoretical_loss": 3.5279601688470006, + "tokens_seen": 1443889152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841123370110331, + "loss": 2.632, + "theoretical_loss": 3.5279459473040626, + "tokens_seen": 1443954688 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841023069207623, + "loss": 2.7038, + "theoretical_loss": 3.527931726587296, + "tokens_seen": 1444020224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840922768304915, + "loss": 2.4187, + "theoretical_loss": 3.527917506696616, + "tokens_seen": 1444085760 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028408224674022066, + "loss": 2.6817, + "theoretical_loss": 3.527903287631937, + "tokens_seen": 1444151296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840722166499499, + "loss": 2.8673, + "theoretical_loss": 3.527889069393174, + "tokens_seen": 1444216832 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284062186559679, + "loss": 2.4415, + "theoretical_loss": 3.5278748519802408, + "tokens_seen": 1444282368 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028405215646940825, + "loss": 2.5479, + "theoretical_loss": 3.527860635393052, + "tokens_seen": 1444347904 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840421263791374, + "loss": 2.7754, + "theoretical_loss": 3.5278464196315227, + "tokens_seen": 1444413440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840320962888666, + "loss": 2.7913, + "theoretical_loss": 3.5278322046955664, + "tokens_seen": 1444478976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840220661985958, + "loss": 2.6492, + "theoretical_loss": 3.527817990585099, + "tokens_seen": 1444544512 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284012036108325, + "loss": 2.4502, + "theoretical_loss": 3.527803777300035, + "tokens_seen": 1444610048 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028400200601805416, + "loss": 2.7361, + "theoretical_loss": 3.5277895648402886, + "tokens_seen": 1444675584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028399197592778334, + "loss": 2.5574, + "theoretical_loss": 3.527775353205774, + "tokens_seen": 1444741120 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839819458375125, + "loss": 2.6315, + "theoretical_loss": 3.5277611423964066, + "tokens_seen": 1444806656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028397191574724176, + "loss": 2.887, + "theoretical_loss": 3.527746932412101, + "tokens_seen": 1444872192 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839618856569709, + "loss": 2.6596, + "theoretical_loss": 3.527732723252771, + "tokens_seen": 1444937728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839518555667001, + "loss": 2.6856, + "theoretical_loss": 3.527718514918332, + "tokens_seen": 1445003264 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 797347, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.670140504837036, + "objective/train/theoretical_loss": 3.5277043074086993, + "objective/train/tokens_used": 1465528800, + "theoretical_loss": 3.5277043074086993, + "tokens_seen": 1445068800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028394182547642925, + "loss": 2.6536, + "theoretical_loss": 3.5277043074086993, + "tokens_seen": 1445068800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839317953861585, + "loss": 2.6025, + "theoretical_loss": 3.527690100723787, + "tokens_seen": 1445134336 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028392176529588766, + "loss": 2.7662, + "theoretical_loss": 3.5276758948635094, + "tokens_seen": 1445199872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028391173520561684, + "loss": 2.6632, + "theoretical_loss": 3.5276616898277817, + "tokens_seen": 1445265408 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283901705115346, + "loss": 2.4511, + "theoretical_loss": 3.5276474856165185, + "tokens_seen": 1445330944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028389167502507526, + "loss": 2.6842, + "theoretical_loss": 3.527633282229635, + "tokens_seen": 1445396480 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838816449348044, + "loss": 2.654, + "theoretical_loss": 3.527619079667046, + "tokens_seen": 1445462016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838716148445336, + "loss": 2.7173, + "theoretical_loss": 3.527604877928665, + "tokens_seen": 1445527552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028386158475426275, + "loss": 2.5172, + "theoretical_loss": 3.5275906770144085, + "tokens_seen": 1445593088 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283851554663992, + "loss": 2.631, + "theoretical_loss": 3.5275764769241906, + "tokens_seen": 1445658624 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028384152457372117, + "loss": 2.5484, + "theoretical_loss": 3.5275622776579256, + "tokens_seen": 1445724160 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028383149448345035, + "loss": 2.618, + "theoretical_loss": 3.527548079215529, + "tokens_seen": 1445789696 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028382146439317953, + "loss": 2.3792, + "theoretical_loss": 3.5275338815969155, + "tokens_seen": 1445855232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838114343029087, + "loss": 2.7984, + "theoretical_loss": 3.5275196848020003, + "tokens_seen": 1445920768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838014042126379, + "loss": 2.5403, + "theoretical_loss": 3.5275054888306974, + "tokens_seen": 1445986304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837913741223671, + "loss": 2.7358, + "theoretical_loss": 3.527491293682923, + "tokens_seen": 1446051840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837813440320963, + "loss": 2.8928, + "theoretical_loss": 3.527477099358591, + "tokens_seen": 1446117376 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837713139418255, + "loss": 2.7105, + "theoretical_loss": 3.5274629058576163, + "tokens_seen": 1446182912 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028376128385155467, + "loss": 2.6156, + "theoretical_loss": 3.527448713179914, + "tokens_seen": 1446248448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028375125376128385, + "loss": 2.4957, + "theoretical_loss": 3.5274345213253993, + "tokens_seen": 1446313984 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837412236710131, + "loss": 2.9047, + "theoretical_loss": 3.527420330293987, + "tokens_seen": 1446379520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837311935807422, + "loss": 2.7489, + "theoretical_loss": 3.527406140085592, + "tokens_seen": 1446445056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028372116349047145, + "loss": 2.7722, + "theoretical_loss": 3.52739195070013, + "tokens_seen": 1446510592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028371113340020063, + "loss": 2.6869, + "theoretical_loss": 3.5273777621375144, + "tokens_seen": 1446576128 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837011033099298, + "loss": 2.6216, + "theoretical_loss": 3.5273635743976617, + "tokens_seen": 1446641664 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 798075, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2298080921173096, + "objective/train/theoretical_loss": 3.527349387480486, + "objective/train/tokens_used": 1467167200, + "theoretical_loss": 3.527349387480486, + "tokens_seen": 1446707200 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283691073219659, + "loss": 2.4326, + "theoretical_loss": 3.527349387480486, + "tokens_seen": 1446707200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836810431293882, + "loss": 2.5288, + "theoretical_loss": 3.5273352013859034, + "tokens_seen": 1446772736 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028367101303911735, + "loss": 2.5907, + "theoretical_loss": 3.5273210161138278, + "tokens_seen": 1446838272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836609829488466, + "loss": 2.7755, + "theoretical_loss": 3.5273068316641747, + "tokens_seen": 1446903808 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836509528585757, + "loss": 2.8434, + "theoretical_loss": 3.5272926480368594, + "tokens_seen": 1446969344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028364092276830495, + "loss": 2.7317, + "theoretical_loss": 3.5272784652317966, + "tokens_seen": 1447034880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836308926780341, + "loss": 2.663, + "theoretical_loss": 3.5272642832489014, + "tokens_seen": 1447100416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836208625877633, + "loss": 2.4391, + "theoretical_loss": 3.5272501020880895, + "tokens_seen": 1447165952 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836108324974925, + "loss": 2.5177, + "theoretical_loss": 3.5272359217492752, + "tokens_seen": 1447231488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836008024072217, + "loss": 2.6772, + "theoretical_loss": 3.527221742232374, + "tokens_seen": 1447297024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028359077231695086, + "loss": 2.5987, + "theoretical_loss": 3.5272075635373015, + "tokens_seen": 1447362560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835807422266801, + "loss": 2.6612, + "theoretical_loss": 3.5271933856639723, + "tokens_seen": 1447428096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835707121364092, + "loss": 2.5096, + "theoretical_loss": 3.5271792086123015, + "tokens_seen": 1447493632 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028356068204613846, + "loss": 2.4976, + "theoretical_loss": 3.5271650323822046, + "tokens_seen": 1447559168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835506519558676, + "loss": 2.8078, + "theoretical_loss": 3.5271508569735968, + "tokens_seen": 1447624704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835406218655968, + "loss": 2.5067, + "theoretical_loss": 3.527136682386393, + "tokens_seen": 1447690240 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283530591775326, + "loss": 2.6418, + "theoretical_loss": 3.5271225086205087, + "tokens_seen": 1447755776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835205616850552, + "loss": 2.6431, + "theoretical_loss": 3.527108335675859, + "tokens_seen": 1447821312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028351053159478436, + "loss": 2.8093, + "theoretical_loss": 3.527094163552359, + "tokens_seen": 1447886848 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028350050150451354, + "loss": 2.4425, + "theoretical_loss": 3.5270799922499245, + "tokens_seen": 1447952384 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834904714142427, + "loss": 2.5708, + "theoretical_loss": 3.5270658217684705, + "tokens_seen": 1448017920 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028348044132397196, + "loss": 2.6207, + "theoretical_loss": 3.5270516521079123, + "tokens_seen": 1448083456 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834704112337011, + "loss": 2.6184, + "theoretical_loss": 3.527037483268165, + "tokens_seen": 1448148992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834603811434303, + "loss": 2.3861, + "theoretical_loss": 3.527023315249144, + "tokens_seen": 1448214528 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028345035105315945, + "loss": 2.7098, + "theoretical_loss": 3.527009148050764, + "tokens_seen": 1448280064 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 799347, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.110480546951294, + "objective/train/theoretical_loss": 3.5269949816729422, + "objective/train/tokens_used": 1468805600, + "theoretical_loss": 3.5269949816729422, + "tokens_seen": 1448345600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834403209628887, + "loss": 2.6722, + "theoretical_loss": 3.5269949816729422, + "tokens_seen": 1448345600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028343029087261786, + "loss": 2.7585, + "theoretical_loss": 3.526980816115592, + "tokens_seen": 1448411136 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028342026078234705, + "loss": 2.7324, + "theoretical_loss": 3.526966651378629, + "tokens_seen": 1448476672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834102306920762, + "loss": 2.7324, + "theoretical_loss": 3.52695248746197, + "tokens_seen": 1448542208 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028340020060180546, + "loss": 2.7503, + "theoretical_loss": 3.5269383243655295, + "tokens_seen": 1448607744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833901705115346, + "loss": 2.7735, + "theoretical_loss": 3.526924162089222, + "tokens_seen": 1448673280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833801404212638, + "loss": 2.6382, + "theoretical_loss": 3.5269100006329643, + "tokens_seen": 1448738816 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028337011033099295, + "loss": 2.6276, + "theoretical_loss": 3.526895839996671, + "tokens_seen": 1448804352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833600802407222, + "loss": 2.6102, + "theoretical_loss": 3.5268816801802583, + "tokens_seen": 1448869888 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028335005015045137, + "loss": 2.5135, + "theoretical_loss": 3.5268675211836404, + "tokens_seen": 1448935424 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028334002006018055, + "loss": 2.711, + "theoretical_loss": 3.526853363006734, + "tokens_seen": 1449000960 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028332998996990973, + "loss": 2.6132, + "theoretical_loss": 3.526839205649454, + "tokens_seen": 1449066496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833199598796389, + "loss": 2.5544, + "theoretical_loss": 3.526825049111716, + "tokens_seen": 1449132032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833099297893681, + "loss": 2.537, + "theoretical_loss": 3.526810893393436, + "tokens_seen": 1449197568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832998996990973, + "loss": 2.6697, + "theoretical_loss": 3.5267967384945287, + "tokens_seen": 1449263104 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028328986960882645, + "loss": 2.58, + "theoretical_loss": 3.526782584414909, + "tokens_seen": 1449328640 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832798395185557, + "loss": 2.75, + "theoretical_loss": 3.526768431154495, + "tokens_seen": 1449394176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832698094282848, + "loss": 2.7828, + "theoretical_loss": 3.5267542787131996, + "tokens_seen": 1449459712 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028325977933801405, + "loss": 2.4813, + "theoretical_loss": 3.526740127090939, + "tokens_seen": 1449525248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028324974924774323, + "loss": 2.8291, + "theoretical_loss": 3.52672597628763, + "tokens_seen": 1449590784 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832397191574724, + "loss": 2.5883, + "theoretical_loss": 3.5267118263031865, + "tokens_seen": 1449656320 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832296890672016, + "loss": 2.8175, + "theoretical_loss": 3.5266976771375256, + "tokens_seen": 1449721856 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028321965897693083, + "loss": 2.5814, + "theoretical_loss": 3.526683528790562, + "tokens_seen": 1449787392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028320962888665996, + "loss": 2.4963, + "theoretical_loss": 3.5266693812622116, + "tokens_seen": 1449852928 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831995987963892, + "loss": 2.5982, + "theoretical_loss": 3.52665523455239, + "tokens_seen": 1449918464 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 800715, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5400002002716064, + "objective/train/theoretical_loss": 3.5266410886610124, + "objective/train/tokens_used": 1470444000, + "theoretical_loss": 3.5266410886610124, + "tokens_seen": 1449984000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831895687061183, + "loss": 2.5756, + "theoretical_loss": 3.5266410886610124, + "tokens_seen": 1449984000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028317953861584755, + "loss": 2.6819, + "theoretical_loss": 3.5266269435879956, + "tokens_seen": 1450049536 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028316950852557674, + "loss": 2.5987, + "theoretical_loss": 3.526612799333254, + "tokens_seen": 1450115072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831594784353059, + "loss": 2.593, + "theoretical_loss": 3.526598655896704, + "tokens_seen": 1450180608 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831494483450351, + "loss": 2.6084, + "theoretical_loss": 3.5265845132782614, + "tokens_seen": 1450246144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831394182547643, + "loss": 2.6047, + "theoretical_loss": 3.526570371477842, + "tokens_seen": 1450311680 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028312938816449346, + "loss": 2.7709, + "theoretical_loss": 3.5265562304953604, + "tokens_seen": 1450377216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831193580742227, + "loss": 2.6368, + "theoretical_loss": 3.526542090330733, + "tokens_seen": 1450442752 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831093279839518, + "loss": 2.6572, + "theoretical_loss": 3.5265279509838763, + "tokens_seen": 1450508288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028309929789368106, + "loss": 2.4918, + "theoretical_loss": 3.5265138124547057, + "tokens_seen": 1450573824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830892678034102, + "loss": 2.5653, + "theoretical_loss": 3.5264996747431363, + "tokens_seen": 1450639360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830792377131394, + "loss": 2.5689, + "theoretical_loss": 3.5264855378490845, + "tokens_seen": 1450704896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830692076228686, + "loss": 2.6756, + "theoretical_loss": 3.526471401772465, + "tokens_seen": 1450770432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830591775325978, + "loss": 2.8919, + "theoretical_loss": 3.5264572665131952, + "tokens_seen": 1450835968 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028304914744232696, + "loss": 2.4432, + "theoretical_loss": 3.5264431320711904, + "tokens_seen": 1450901504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830391173520562, + "loss": 2.7723, + "theoretical_loss": 3.526428998446366, + "tokens_seen": 1450967040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830290872617854, + "loss": 2.7163, + "theoretical_loss": 3.5264148656386385, + "tokens_seen": 1451032576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028301905717151456, + "loss": 2.6018, + "theoretical_loss": 3.526400733647923, + "tokens_seen": 1451098112 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028300902708124374, + "loss": 2.5447, + "theoretical_loss": 3.526386602474136, + "tokens_seen": 1451163648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829989969909729, + "loss": 2.6576, + "theoretical_loss": 3.5263724721171927, + "tokens_seen": 1451229184 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028298896690070216, + "loss": 2.4897, + "theoretical_loss": 3.5263583425770095, + "tokens_seen": 1451294720 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829789368104313, + "loss": 2.4394, + "theoretical_loss": 3.526344213853503, + "tokens_seen": 1451360256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829689067201605, + "loss": 2.7323, + "theoretical_loss": 3.5263300859465874, + "tokens_seen": 1451425792 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028295887662988965, + "loss": 2.6551, + "theoretical_loss": 3.52631595885618, + "tokens_seen": 1451491328 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829488465396189, + "loss": 2.5651, + "theoretical_loss": 3.526301832582196, + "tokens_seen": 1451556864 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 801482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5024585723876953, + "objective/train/theoretical_loss": 3.5262877071245526, + "objective/train/tokens_used": 1472082400, + "theoretical_loss": 3.5262877071245526, + "tokens_seen": 1451622400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028293881644934806, + "loss": 2.6494, + "theoretical_loss": 3.5262877071245526, + "tokens_seen": 1451622400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028292878635907725, + "loss": 2.6545, + "theoretical_loss": 3.526273582483164, + "tokens_seen": 1451687936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829187562688064, + "loss": 2.3686, + "theoretical_loss": 3.5262594586579477, + "tokens_seen": 1451753472 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028290872617853566, + "loss": 2.4083, + "theoretical_loss": 3.5262453356488184, + "tokens_seen": 1451819008 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828986960882648, + "loss": 2.9027, + "theoretical_loss": 3.5262312134556932, + "tokens_seen": 1451884544 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282888665997994, + "loss": 2.4601, + "theoretical_loss": 3.526217092078488, + "tokens_seen": 1451950080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028287863590772315, + "loss": 2.7495, + "theoretical_loss": 3.526202971517118, + "tokens_seen": 1452015616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828686058174524, + "loss": 2.3887, + "theoretical_loss": 3.5261888517715, + "tokens_seen": 1452081152 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028285857572718157, + "loss": 2.4065, + "theoretical_loss": 3.52617473284155, + "tokens_seen": 1452146688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028284854563691075, + "loss": 2.8447, + "theoretical_loss": 3.526160614727184, + "tokens_seen": 1452212224 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028283851554663993, + "loss": 2.7958, + "theoretical_loss": 3.526146497428317, + "tokens_seen": 1452277760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828284854563691, + "loss": 2.6396, + "theoretical_loss": 3.5261323809448673, + "tokens_seen": 1452343296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828184553660983, + "loss": 2.7114, + "theoretical_loss": 3.5261182652767493, + "tokens_seen": 1452408832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828084252758275, + "loss": 2.6084, + "theoretical_loss": 3.5261041504238797, + "tokens_seen": 1452474368 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028279839518555665, + "loss": 2.4572, + "theoretical_loss": 3.526090036386174, + "tokens_seen": 1452539904 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827883650952859, + "loss": 2.681, + "theoretical_loss": 3.5260759231635497, + "tokens_seen": 1452605440 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282778335005015, + "loss": 2.6388, + "theoretical_loss": 3.526061810755922, + "tokens_seen": 1452670976 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028276830491474425, + "loss": 2.5954, + "theoretical_loss": 3.526047699163207, + "tokens_seen": 1452736512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028275827482447343, + "loss": 2.5933, + "theoretical_loss": 3.5260335883853213, + "tokens_seen": 1452802048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827482447342026, + "loss": 2.5419, + "theoretical_loss": 3.5260194784221808, + "tokens_seen": 1452867584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827382146439318, + "loss": 2.3594, + "theoretical_loss": 3.526005369273702, + "tokens_seen": 1452933120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028272818455366103, + "loss": 2.7973, + "theoretical_loss": 3.5259912609398008, + "tokens_seen": 1452998656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028271815446339016, + "loss": 2.6136, + "theoretical_loss": 3.525977153420394, + "tokens_seen": 1453064192 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827081243731194, + "loss": 2.5128, + "theoretical_loss": 3.525963046715397, + "tokens_seen": 1453129728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826980942828485, + "loss": 2.5333, + "theoretical_loss": 3.525948940824726, + "tokens_seen": 1453195264 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 802380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2598981857299805, + "objective/train/theoretical_loss": 3.5259348357482985, + "objective/train/tokens_used": 1473720800, + "theoretical_loss": 3.5259348357482985, + "tokens_seen": 1453260800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028268806419257775, + "loss": 2.4851, + "theoretical_loss": 3.5259348357482985, + "tokens_seen": 1453260800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028267803410230694, + "loss": 2.4297, + "theoretical_loss": 3.52592073148603, + "tokens_seen": 1453326336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826680040120361, + "loss": 2.6273, + "theoretical_loss": 3.525906628037837, + "tokens_seen": 1453391872 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826579739217653, + "loss": 2.5379, + "theoretical_loss": 3.525892525403635, + "tokens_seen": 1453457408 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826479438314945, + "loss": 2.5902, + "theoretical_loss": 3.5258784235833414, + "tokens_seen": 1453522944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028263791374122366, + "loss": 2.6003, + "theoretical_loss": 3.525864322576872, + "tokens_seen": 1453588480 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826278836509529, + "loss": 2.7424, + "theoretical_loss": 3.5258502223841433, + "tokens_seen": 1453654016 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282617853560682, + "loss": 2.9454, + "theoretical_loss": 3.5258361230050714, + "tokens_seen": 1453719552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028260782347041126, + "loss": 2.6894, + "theoretical_loss": 3.525822024439573, + "tokens_seen": 1453785088 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825977933801404, + "loss": 2.6092, + "theoretical_loss": 3.5258079266875644, + "tokens_seen": 1453850624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825877632898696, + "loss": 2.5426, + "theoretical_loss": 3.525793829748962, + "tokens_seen": 1453916160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825777331995988, + "loss": 2.6994, + "theoretical_loss": 3.5257797336236822, + "tokens_seen": 1453981696 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282567703109328, + "loss": 2.4356, + "theoretical_loss": 3.525765638311641, + "tokens_seen": 1454047232 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028255767301905716, + "loss": 2.681, + "theoretical_loss": 3.5257515438127554, + "tokens_seen": 1454112768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825476429287864, + "loss": 2.6066, + "theoretical_loss": 3.5257374501269423, + "tokens_seen": 1454178304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825376128385155, + "loss": 2.7642, + "theoretical_loss": 3.5257233572541167, + "tokens_seen": 1454243840 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028252758274824476, + "loss": 2.6863, + "theoretical_loss": 3.525709265194196, + "tokens_seen": 1454309376 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825175526579739, + "loss": 2.657, + "theoretical_loss": 3.5256951739470965, + "tokens_seen": 1454374912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825075225677031, + "loss": 2.8041, + "theoretical_loss": 3.525681083512735, + "tokens_seen": 1454440448 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824974924774323, + "loss": 2.5356, + "theoretical_loss": 3.5256669938910274, + "tokens_seen": 1454505984 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824874623871615, + "loss": 2.7467, + "theoretical_loss": 3.525652905081891, + "tokens_seen": 1454571520 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028247743229689067, + "loss": 2.8853, + "theoretical_loss": 3.525638817085242, + "tokens_seen": 1454637056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028246740220661985, + "loss": 2.786, + "theoretical_loss": 3.525624729900996, + "tokens_seen": 1454702592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028245737211634903, + "loss": 2.7916, + "theoretical_loss": 3.5256106435290713, + "tokens_seen": 1454768128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028244734202607826, + "loss": 2.6602, + "theoretical_loss": 3.5255965579693838, + "tokens_seen": 1454833664 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 802899, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6358089447021484, + "objective/train/theoretical_loss": 3.525582473221849, + "objective/train/tokens_used": 1475359200, + "theoretical_loss": 3.525582473221849, + "tokens_seen": 1454899200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824373119358074, + "loss": 2.8029, + "theoretical_loss": 3.525582473221849, + "tokens_seen": 1454899200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824272818455366, + "loss": 2.6266, + "theoretical_loss": 3.5255683892863843, + "tokens_seen": 1454964736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824172517552658, + "loss": 2.4783, + "theoretical_loss": 3.525554306162907, + "tokens_seen": 1455030272 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282407221664995, + "loss": 2.5727, + "theoretical_loss": 3.5255402238513325, + "tokens_seen": 1455095808 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028239719157472417, + "loss": 2.6995, + "theoretical_loss": 3.5255261423515782, + "tokens_seen": 1455161344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028238716148445335, + "loss": 2.6435, + "theoretical_loss": 3.525512061663561, + "tokens_seen": 1455226880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028237713139418253, + "loss": 2.7177, + "theoretical_loss": 3.5254979817871965, + "tokens_seen": 1455292416 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028236710130391177, + "loss": 2.7309, + "theoretical_loss": 3.525483902722402, + "tokens_seen": 1455357952 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823570712136409, + "loss": 2.7335, + "theoretical_loss": 3.525469824469094, + "tokens_seen": 1455423488 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028234704112337013, + "loss": 2.6619, + "theoretical_loss": 3.52545574702719, + "tokens_seen": 1455489024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028233701103309926, + "loss": 2.5114, + "theoretical_loss": 3.5254416703966056, + "tokens_seen": 1455554560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823269809428285, + "loss": 2.7022, + "theoretical_loss": 3.5254275945772577, + "tokens_seen": 1455620096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823169508525577, + "loss": 2.8818, + "theoretical_loss": 3.5254135195690637, + "tokens_seen": 1455685632 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028230692076228685, + "loss": 2.5581, + "theoretical_loss": 3.5253994453719395, + "tokens_seen": 1455751168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028229689067201604, + "loss": 2.8764, + "theoretical_loss": 3.525385371985802, + "tokens_seen": 1455816704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822868605817452, + "loss": 2.7061, + "theoretical_loss": 3.525371299410569, + "tokens_seen": 1455882240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028227683049147445, + "loss": 2.6916, + "theoretical_loss": 3.525357227646156, + "tokens_seen": 1455947776 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028226680040120363, + "loss": 2.6838, + "theoretical_loss": 3.5253431566924807, + "tokens_seen": 1456013312 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822567703109328, + "loss": 2.6026, + "theoretical_loss": 3.5253290865494593, + "tokens_seen": 1456078848 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282246740220662, + "loss": 2.5756, + "theoretical_loss": 3.5253150172170087, + "tokens_seen": 1456144384 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028223671013039123, + "loss": 2.5553, + "theoretical_loss": 3.5253009486950457, + "tokens_seen": 1456209920 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028222668004012036, + "loss": 2.589, + "theoretical_loss": 3.525286880983488, + "tokens_seen": 1456275456 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822166499498496, + "loss": 2.7555, + "theoretical_loss": 3.5252728140822507, + "tokens_seen": 1456340992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822066198595787, + "loss": 2.5639, + "theoretical_loss": 3.525258747991252, + "tokens_seen": 1456406528 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028219658976930795, + "loss": 2.6847, + "theoretical_loss": 3.525244682710409, + "tokens_seen": 1456472064 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 804182, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6662416458129883, + "objective/train/theoretical_loss": 3.525230618239638, + "objective/train/tokens_used": 1476997600, + "theoretical_loss": 3.525230618239638, + "tokens_seen": 1456537600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028218655967903714, + "loss": 2.5008, + "theoretical_loss": 3.525230618239638, + "tokens_seen": 1456537600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821765295887663, + "loss": 2.5923, + "theoretical_loss": 3.525216554578855, + "tokens_seen": 1456603136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821664994984955, + "loss": 2.7399, + "theoretical_loss": 3.5252024917279785, + "tokens_seen": 1456668672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821564694082247, + "loss": 2.8464, + "theoretical_loss": 3.525188429686925, + "tokens_seen": 1456734208 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028214643931795386, + "loss": 2.6559, + "theoretical_loss": 3.5251743684556107, + "tokens_seen": 1456799744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821364092276831, + "loss": 2.8568, + "theoretical_loss": 3.5251603080339535, + "tokens_seen": 1456865280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821263791374122, + "loss": 2.468, + "theoretical_loss": 3.525146248421869, + "tokens_seen": 1456930816 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028211634904714146, + "loss": 2.6813, + "theoretical_loss": 3.5251321896192764, + "tokens_seen": 1456996352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821063189568706, + "loss": 2.7486, + "theoretical_loss": 3.5251181316260904, + "tokens_seen": 1457061888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820962888665998, + "loss": 2.3982, + "theoretical_loss": 3.5251040744422295, + "tokens_seen": 1457127424 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282086258776329, + "loss": 2.6629, + "theoretical_loss": 3.52509001806761, + "tokens_seen": 1457192960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820762286860582, + "loss": 2.6004, + "theoretical_loss": 3.5250759625021493, + "tokens_seen": 1457258496 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028206619859578736, + "loss": 2.6276, + "theoretical_loss": 3.525061907745764, + "tokens_seen": 1457324032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820561685055166, + "loss": 2.7723, + "theoretical_loss": 3.5250478537983714, + "tokens_seen": 1457389568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820461384152457, + "loss": 2.5502, + "theoretical_loss": 3.5250338006598887, + "tokens_seen": 1457455104 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028203610832497496, + "loss": 2.6508, + "theoretical_loss": 3.525019748330233, + "tokens_seen": 1457520640 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820260782347041, + "loss": 2.5532, + "theoretical_loss": 3.5250056968093206, + "tokens_seen": 1457586176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820160481444333, + "loss": 2.647, + "theoretical_loss": 3.5249916460970696, + "tokens_seen": 1457651712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820060180541625, + "loss": 2.5241, + "theoretical_loss": 3.5249775961933967, + "tokens_seen": 1457717248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819959879638917, + "loss": 2.5282, + "theoretical_loss": 3.524963547098219, + "tokens_seen": 1457782784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028198595787362087, + "loss": 2.6907, + "theoretical_loss": 3.524949498811454, + "tokens_seen": 1457848320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028197592778335005, + "loss": 2.7418, + "theoretical_loss": 3.5249354513330178, + "tokens_seen": 1457913856 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028196589769307923, + "loss": 2.4679, + "theoretical_loss": 3.5249214046628286, + "tokens_seen": 1457979392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028195586760280846, + "loss": 2.6644, + "theoretical_loss": 3.524907358800803, + "tokens_seen": 1458044928 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819458375125376, + "loss": 2.8422, + "theoretical_loss": 3.5248933137468588, + "tokens_seen": 1458110464 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 804616, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4237711429595947, + "objective/train/theoretical_loss": 3.5248792695009126, + "objective/train/tokens_used": 1478636000, + "theoretical_loss": 3.5248792695009126, + "tokens_seen": 1458176000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819358074222668, + "loss": 2.691, + "theoretical_loss": 3.5248792695009126, + "tokens_seen": 1458176000 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281925777331996, + "loss": 2.9381, + "theoretical_loss": 3.5248652260628814, + "tokens_seen": 1458241536 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819157472417252, + "loss": 2.7701, + "theoretical_loss": 3.5248511834326837, + "tokens_seen": 1458307072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028190571715145437, + "loss": 2.7494, + "theoretical_loss": 3.5248371416102353, + "tokens_seen": 1458372608 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028189568706118355, + "loss": 2.8327, + "theoretical_loss": 3.5248231005954542, + "tokens_seen": 1458438144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028188565697091273, + "loss": 2.5657, + "theoretical_loss": 3.5248090603882574, + "tokens_seen": 1458503680 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028187562688064197, + "loss": 2.7865, + "theoretical_loss": 3.524795020988562, + "tokens_seen": 1458569216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818655967903711, + "loss": 2.5236, + "theoretical_loss": 3.5247809823962855, + "tokens_seen": 1458634752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028185556670010033, + "loss": 2.5228, + "theoretical_loss": 3.524766944611345, + "tokens_seen": 1458700288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028184553660982946, + "loss": 2.6959, + "theoretical_loss": 3.524752907633659, + "tokens_seen": 1458765824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818355065195587, + "loss": 2.6638, + "theoretical_loss": 3.5247388714631427, + "tokens_seen": 1458831360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818254764292879, + "loss": 2.5239, + "theoretical_loss": 3.5247248360997148, + "tokens_seen": 1458896896 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028181544633901705, + "loss": 2.58, + "theoretical_loss": 3.524710801543293, + "tokens_seen": 1458962432 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028180541624874624, + "loss": 2.3986, + "theoretical_loss": 3.5246967677937935, + "tokens_seen": 1459027968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817953861584754, + "loss": 2.6534, + "theoretical_loss": 3.524682734851134, + "tokens_seen": 1459093504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817853560682046, + "loss": 2.6836, + "theoretical_loss": 3.524668702715232, + "tokens_seen": 1459159040 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028177532597793383, + "loss": 2.7844, + "theoretical_loss": 3.5246546713860054, + "tokens_seen": 1459224576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028176529588766296, + "loss": 2.4228, + "theoretical_loss": 3.524640640863371, + "tokens_seen": 1459290112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817552657973922, + "loss": 2.5844, + "theoretical_loss": 3.524626611147246, + "tokens_seen": 1459355648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817452357071214, + "loss": 2.443, + "theoretical_loss": 3.524612582237549, + "tokens_seen": 1459421184 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028173520561685056, + "loss": 2.8536, + "theoretical_loss": 3.524598554134196, + "tokens_seen": 1459486720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028172517552657974, + "loss": 2.5898, + "theoretical_loss": 3.5245845268371054, + "tokens_seen": 1459552256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817151454363089, + "loss": 2.8134, + "theoretical_loss": 3.5245705003461936, + "tokens_seen": 1459617792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817051153460381, + "loss": 2.8013, + "theoretical_loss": 3.5245564746613796, + "tokens_seen": 1459683328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028169508525576734, + "loss": 2.7783, + "theoretical_loss": 3.52454244978258, + "tokens_seen": 1459748864 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 805828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6058542728424072, + "objective/train/theoretical_loss": 3.524528425709712, + "objective/train/tokens_used": 1480274400, + "theoretical_loss": 3.524528425709712, + "tokens_seen": 1459814400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028168505516549646, + "loss": 2.7628, + "theoretical_loss": 3.524528425709712, + "tokens_seen": 1459814400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816750250752257, + "loss": 2.6103, + "theoretical_loss": 3.5245144024426938, + "tokens_seen": 1459879936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816649949849548, + "loss": 2.7047, + "theoretical_loss": 3.5245003799814425, + "tokens_seen": 1459945472 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028165496489468406, + "loss": 2.6731, + "theoretical_loss": 3.5244863583258756, + "tokens_seen": 1460011008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028164493480441324, + "loss": 2.9568, + "theoretical_loss": 3.524472337475911, + "tokens_seen": 1460076544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816349047141424, + "loss": 2.7297, + "theoretical_loss": 3.524458317431466, + "tokens_seen": 1460142080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816248746238716, + "loss": 2.501, + "theoretical_loss": 3.524444298192458, + "tokens_seen": 1460207616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816148445336008, + "loss": 2.7512, + "theoretical_loss": 3.524430279758805, + "tokens_seen": 1460273152 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028160481444332997, + "loss": 2.6057, + "theoretical_loss": 3.5244162621304245, + "tokens_seen": 1460338688 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815947843530592, + "loss": 2.6087, + "theoretical_loss": 3.5244022453072343, + "tokens_seen": 1460404224 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028158475426278833, + "loss": 2.7053, + "theoretical_loss": 3.5243882292891513, + "tokens_seen": 1460469760 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028157472417251756, + "loss": 2.7984, + "theoretical_loss": 3.5243742140760936, + "tokens_seen": 1460535296 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028156469408224674, + "loss": 2.5999, + "theoretical_loss": 3.5243601996679788, + "tokens_seen": 1460600832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815546639919759, + "loss": 2.4843, + "theoretical_loss": 3.524346186064725, + "tokens_seen": 1460666368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815446339017051, + "loss": 2.8821, + "theoretical_loss": 3.5243321732662487, + "tokens_seen": 1460731904 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815346038114343, + "loss": 2.413, + "theoretical_loss": 3.5243181612724688, + "tokens_seen": 1460797440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815245737211635, + "loss": 2.6267, + "theoretical_loss": 3.5243041500833026, + "tokens_seen": 1460862976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815145436308927, + "loss": 2.5767, + "theoretical_loss": 3.5242901396986674, + "tokens_seen": 1460928512 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815045135406219, + "loss": 2.6285, + "theoretical_loss": 3.524276130118481, + "tokens_seen": 1460994048 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028149448345035107, + "loss": 2.464, + "theoretical_loss": 3.5242621213426615, + "tokens_seen": 1461059584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028148445336008025, + "loss": 2.6859, + "theoretical_loss": 3.5242481133711268, + "tokens_seen": 1461125120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028147442326980943, + "loss": 2.6109, + "theoretical_loss": 3.524234106203794, + "tokens_seen": 1461190656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028146439317953866, + "loss": 2.6543, + "theoretical_loss": 3.5242200998405817, + "tokens_seen": 1461256192 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814543630892678, + "loss": 2.8038, + "theoretical_loss": 3.5242060942814066, + "tokens_seen": 1461321728 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281444332998997, + "loss": 2.5912, + "theoretical_loss": 3.5241920895261876, + "tokens_seen": 1461387264 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 806380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.800405740737915, + "objective/train/theoretical_loss": 3.5241780855748415, + "objective/train/tokens_used": 1481912800, + "theoretical_loss": 3.5241780855748415, + "tokens_seen": 1461452800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814343029087262, + "loss": 2.5523, + "theoretical_loss": 3.5241780855748415, + "tokens_seen": 1461452800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814242728184554, + "loss": 2.8003, + "theoretical_loss": 3.5241640824272866, + "tokens_seen": 1461518336 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028141424272818457, + "loss": 2.5437, + "theoretical_loss": 3.524150080083441, + "tokens_seen": 1461583872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028140421263791375, + "loss": 2.6448, + "theoretical_loss": 3.524136078543222, + "tokens_seen": 1461649408 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028139418254764293, + "loss": 2.5327, + "theoretical_loss": 3.5241220778065476, + "tokens_seen": 1461714944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028138415245737217, + "loss": 2.8325, + "theoretical_loss": 3.5241080778733362, + "tokens_seen": 1461780480 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813741223671013, + "loss": 2.6118, + "theoretical_loss": 3.524094078743505, + "tokens_seen": 1461846016 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028136409227683053, + "loss": 2.4199, + "theoretical_loss": 3.524080080416972, + "tokens_seen": 1461911552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028135406218655966, + "loss": 2.6473, + "theoretical_loss": 3.524066082893655, + "tokens_seen": 1461977088 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813440320962889, + "loss": 2.4552, + "theoretical_loss": 3.5240520861734725, + "tokens_seen": 1462042624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813340020060181, + "loss": 2.7012, + "theoretical_loss": 3.524038090256342, + "tokens_seen": 1462108160 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028132397191574725, + "loss": 2.5372, + "theoretical_loss": 3.524024095142181, + "tokens_seen": 1462173696 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028131394182547644, + "loss": 2.5762, + "theoretical_loss": 3.5240101008309086, + "tokens_seen": 1462239232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813039117352056, + "loss": 2.8869, + "theoretical_loss": 3.523996107322442, + "tokens_seen": 1462304768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812938816449348, + "loss": 2.7317, + "theoretical_loss": 3.5239821146166985, + "tokens_seen": 1462370304 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028128385155466403, + "loss": 2.5509, + "theoretical_loss": 3.5239681227135975, + "tokens_seen": 1462435840 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028127382146439316, + "loss": 2.6265, + "theoretical_loss": 3.5239541316130563, + "tokens_seen": 1462501376 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812637913741224, + "loss": 2.6827, + "theoretical_loss": 3.5239401413149927, + "tokens_seen": 1462566912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812537612838516, + "loss": 2.7779, + "theoretical_loss": 3.523926151819325, + "tokens_seen": 1462632448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028124373119358076, + "loss": 2.5768, + "theoretical_loss": 3.523912163125971, + "tokens_seen": 1462697984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028123370110330994, + "loss": 2.6157, + "theoretical_loss": 3.5238981752348493, + "tokens_seen": 1462763520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812236710130391, + "loss": 2.8139, + "theoretical_loss": 3.5238841881458773, + "tokens_seen": 1462829056 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812136409227683, + "loss": 2.5889, + "theoretical_loss": 3.5238702018589736, + "tokens_seen": 1462894592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028120361083249754, + "loss": 2.5345, + "theoretical_loss": 3.523856216374056, + "tokens_seen": 1462960128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028119358074222666, + "loss": 2.6998, + "theoretical_loss": 3.5238422316910425, + "tokens_seen": 1463025664 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 807855, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4178123474121094, + "objective/train/theoretical_loss": 3.523828247809851, + "objective/train/tokens_used": 1483551200, + "theoretical_loss": 3.523828247809851, + "tokens_seen": 1463091200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811835506519559, + "loss": 2.4975, + "theoretical_loss": 3.523828247809851, + "tokens_seen": 1463091200 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281173520561685, + "loss": 2.6831, + "theoretical_loss": 3.5238142647304, + "tokens_seen": 1463156736 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028116349047141426, + "loss": 2.7051, + "theoretical_loss": 3.5238002824526076, + "tokens_seen": 1463222272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028115346038114344, + "loss": 2.4668, + "theoretical_loss": 3.5237863009763917, + "tokens_seen": 1463287808 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811434302908726, + "loss": 2.5234, + "theoretical_loss": 3.523772320301671, + "tokens_seen": 1463353344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811334002006018, + "loss": 2.4375, + "theoretical_loss": 3.5237583404283637, + "tokens_seen": 1463418880 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281123370110331, + "loss": 2.5769, + "theoretical_loss": 3.523744361356387, + "tokens_seen": 1463484416 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028111334002006017, + "loss": 2.6807, + "theoretical_loss": 3.5237303830856597, + "tokens_seen": 1463549952 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811033099297894, + "loss": 2.6226, + "theoretical_loss": 3.5237164056160997, + "tokens_seen": 1463615488 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028109327983951853, + "loss": 2.5343, + "theoretical_loss": 3.523702428947626, + "tokens_seen": 1463681024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028108324974924776, + "loss": 2.6531, + "theoretical_loss": 3.5236884530801555, + "tokens_seen": 1463746560 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028107321965897695, + "loss": 2.683, + "theoretical_loss": 3.523674478013608, + "tokens_seen": 1463812096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810631895687061, + "loss": 2.8166, + "theoretical_loss": 3.5236605037479007, + "tokens_seen": 1463877632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810531594784353, + "loss": 2.494, + "theoretical_loss": 3.523646530282952, + "tokens_seen": 1463943168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810431293881645, + "loss": 2.5355, + "theoretical_loss": 3.52363255761868, + "tokens_seen": 1464008704 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028103309929789367, + "loss": 2.395, + "theoretical_loss": 3.5236185857550044, + "tokens_seen": 1464074240 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810230692076229, + "loss": 2.5215, + "theoretical_loss": 3.523604614691841, + "tokens_seen": 1464139776 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028101303911735203, + "loss": 2.5591, + "theoretical_loss": 3.5235906444291105, + "tokens_seen": 1464205312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028100300902708127, + "loss": 2.5624, + "theoretical_loss": 3.52357667496673, + "tokens_seen": 1464270848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002809929789368104, + "loss": 2.5935, + "theoretical_loss": 3.5235627063046175, + "tokens_seen": 1464336384 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028098294884653963, + "loss": 2.8998, + "theoretical_loss": 3.5235487384426927, + "tokens_seen": 1464401920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002809729187562688, + "loss": 2.5753, + "theoretical_loss": 3.5235347713808727, + "tokens_seen": 1464467456 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280962888665998, + "loss": 2.5725, + "theoretical_loss": 3.523520805119076, + "tokens_seen": 1464532992 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028095285857572717, + "loss": 2.7663, + "theoretical_loss": 3.5235068396572213, + "tokens_seen": 1464598528 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028094282848545635, + "loss": 2.6161, + "theoretical_loss": 3.5234928749952275, + "tokens_seen": 1464664064 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 808908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.759751319885254, + "objective/train/theoretical_loss": 3.523478911133012, + "objective/train/tokens_used": 1485189600, + "theoretical_loss": 3.523478911133012, + "tokens_seen": 1464729600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028093279839518553, + "loss": 2.6224, + "theoretical_loss": 3.523478911133012, + "tokens_seen": 1464729600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028092276830491477, + "loss": 2.7913, + "theoretical_loss": 3.523464948070494, + "tokens_seen": 1464795136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002809127382146439, + "loss": 2.7087, + "theoretical_loss": 3.5234509858075915, + "tokens_seen": 1464860672 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028090270812437313, + "loss": 2.6517, + "theoretical_loss": 3.5234370243442226, + "tokens_seen": 1464926208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808926780341023, + "loss": 2.6858, + "theoretical_loss": 3.5234230636803066, + "tokens_seen": 1464991744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808826479438315, + "loss": 2.6047, + "theoretical_loss": 3.5234091038157613, + "tokens_seen": 1465057280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808726178535607, + "loss": 2.6421, + "theoretical_loss": 3.5233951447505056, + "tokens_seen": 1465122816 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028086258776328986, + "loss": 2.443, + "theoretical_loss": 3.5233811864844577, + "tokens_seen": 1465188352 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028085255767301904, + "loss": 2.8726, + "theoretical_loss": 3.523367229017537, + "tokens_seen": 1465253888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808425275827483, + "loss": 2.7869, + "theoretical_loss": 3.5233532723496603, + "tokens_seen": 1465319424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808324974924774, + "loss": 2.5656, + "theoretical_loss": 3.523339316480747, + "tokens_seen": 1465384960 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028082246740220664, + "loss": 2.5999, + "theoretical_loss": 3.5233253614107163, + "tokens_seen": 1465450496 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028081243731193576, + "loss": 2.4881, + "theoretical_loss": 3.5233114071394853, + "tokens_seen": 1465516032 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280802407221665, + "loss": 2.5154, + "theoretical_loss": 3.5232974536669737, + "tokens_seen": 1465581568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807923771313942, + "loss": 2.5077, + "theoretical_loss": 3.5232835009931005, + "tokens_seen": 1465647104 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028078234704112336, + "loss": 2.6592, + "theoretical_loss": 3.523269549117783, + "tokens_seen": 1465712640 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807723169508526, + "loss": 2.4471, + "theoretical_loss": 3.52325559804094, + "tokens_seen": 1465778176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807622868605818, + "loss": 2.5608, + "theoretical_loss": 3.5232416477624904, + "tokens_seen": 1465843712 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028075225677031096, + "loss": 2.4759, + "theoretical_loss": 3.523227698282353, + "tokens_seen": 1465909248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028074222668004014, + "loss": 2.4707, + "theoretical_loss": 3.5232137496004468, + "tokens_seen": 1465974784 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807321965897693, + "loss": 2.5881, + "theoretical_loss": 3.5231998017166894, + "tokens_seen": 1466040320 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807221664994985, + "loss": 2.593, + "theoretical_loss": 3.523185854631, + "tokens_seen": 1466105856 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028071213640922774, + "loss": 2.7124, + "theoretical_loss": 3.523171908343297, + "tokens_seen": 1466171392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028070210631895686, + "loss": 2.4851, + "theoretical_loss": 3.5231579628535004, + "tokens_seen": 1466236928 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806920762286861, + "loss": 2.6762, + "theoretical_loss": 3.523144018161527, + "tokens_seen": 1466302464 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 809565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5152714252471924, + "objective/train/theoretical_loss": 3.523130074267296, + "objective/train/tokens_used": 1486828000, + "theoretical_loss": 3.523130074267296, + "tokens_seen": 1466368000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806820461384152, + "loss": 2.5798, + "theoretical_loss": 3.523130074267296, + "tokens_seen": 1466368000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028067201604814446, + "loss": 2.6714, + "theoretical_loss": 3.5231161311707266, + "tokens_seen": 1466433536 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028066198595787364, + "loss": 2.7615, + "theoretical_loss": 3.5231021888717375, + "tokens_seen": 1466499072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806519558676028, + "loss": 2.6979, + "theoretical_loss": 3.5230882473702474, + "tokens_seen": 1466564608 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280641925777332, + "loss": 2.7179, + "theoretical_loss": 3.5230743066661745, + "tokens_seen": 1466630144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806318956870612, + "loss": 2.4576, + "theoretical_loss": 3.523060366759438, + "tokens_seen": 1466695680 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028062186559679037, + "loss": 2.6031, + "theoretical_loss": 3.523046427649957, + "tokens_seen": 1466761216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806118355065196, + "loss": 2.5807, + "theoretical_loss": 3.52303248933765, + "tokens_seen": 1466826752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028060180541624873, + "loss": 2.7375, + "theoretical_loss": 3.5230185518224357, + "tokens_seen": 1466892288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028059177532597796, + "loss": 2.6703, + "theoretical_loss": 3.5230046151042327, + "tokens_seen": 1466957824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028058174523570715, + "loss": 2.5978, + "theoretical_loss": 3.5229906791829606, + "tokens_seen": 1467023360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805717151454363, + "loss": 2.6568, + "theoretical_loss": 3.522976744058537, + "tokens_seen": 1467088896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805616850551655, + "loss": 2.5362, + "theoretical_loss": 3.522962809730882, + "tokens_seen": 1467154432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805516549648947, + "loss": 2.6053, + "theoretical_loss": 3.522948876199913, + "tokens_seen": 1467219968 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028054162487462387, + "loss": 2.448, + "theoretical_loss": 3.5229349434655504, + "tokens_seen": 1467285504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805315947843531, + "loss": 2.7329, + "theoretical_loss": 3.5229210115277128, + "tokens_seen": 1467351040 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028052156469408223, + "loss": 2.649, + "theoretical_loss": 3.522907080386318, + "tokens_seen": 1467416576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028051153460381147, + "loss": 2.5478, + "theoretical_loss": 3.5228931500412863, + "tokens_seen": 1467482112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805015045135406, + "loss": 2.5681, + "theoretical_loss": 3.522879220492536, + "tokens_seen": 1467547648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028049147442326983, + "loss": 2.5845, + "theoretical_loss": 3.5228652917399854, + "tokens_seen": 1467613184 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280481444332999, + "loss": 2.613, + "theoretical_loss": 3.5228513637835537, + "tokens_seen": 1467678720 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804714142427282, + "loss": 2.6921, + "theoretical_loss": 3.522837436623161, + "tokens_seen": 1467744256 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028046138415245737, + "loss": 2.5427, + "theoretical_loss": 3.5228235102587253, + "tokens_seen": 1467809792 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028045135406218655, + "loss": 2.6837, + "theoretical_loss": 3.5228095846901653, + "tokens_seen": 1467875328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028044132397191574, + "loss": 2.7453, + "theoretical_loss": 3.522795659917401, + "tokens_seen": 1467940864 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 810763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.383155345916748, + "objective/train/theoretical_loss": 3.5227817359403506, + "objective/train/tokens_used": 1488466400, + "theoretical_loss": 3.5227817359403506, + "tokens_seen": 1468006400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028043129388164497, + "loss": 2.4052, + "theoretical_loss": 3.5227817359403506, + "tokens_seen": 1468006400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804212637913741, + "loss": 2.7047, + "theoretical_loss": 3.522767812758933, + "tokens_seen": 1468071936 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028041123370110333, + "loss": 2.646, + "theoretical_loss": 3.5227538903730675, + "tokens_seen": 1468137472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804012036108325, + "loss": 2.5549, + "theoretical_loss": 3.5227399687826733, + "tokens_seen": 1468203008 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002803911735205617, + "loss": 2.5508, + "theoretical_loss": 3.5227260479876694, + "tokens_seen": 1468268544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002803811434302909, + "loss": 2.561, + "theoretical_loss": 3.5227121279879743, + "tokens_seen": 1468334080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028037111334002006, + "loss": 2.5237, + "theoretical_loss": 3.522698208783508, + "tokens_seen": 1468399616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028036108324974924, + "loss": 2.4907, + "theoretical_loss": 3.522684290374189, + "tokens_seen": 1468465152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803510531594785, + "loss": 2.5964, + "theoretical_loss": 3.5226703727599364, + "tokens_seen": 1468530688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803410230692076, + "loss": 2.6129, + "theoretical_loss": 3.5226564559406697, + "tokens_seen": 1468596224 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028033099297893684, + "loss": 2.3882, + "theoretical_loss": 3.5226425399163075, + "tokens_seen": 1468661760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028032096288866596, + "loss": 2.753, + "theoretical_loss": 3.522628624686769, + "tokens_seen": 1468727296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803109327983952, + "loss": 2.5923, + "theoretical_loss": 3.5226147102519736, + "tokens_seen": 1468792832 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803009027081244, + "loss": 2.5226, + "theoretical_loss": 3.5226007966118407, + "tokens_seen": 1468858368 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028029087261785356, + "loss": 2.919, + "theoretical_loss": 3.522586883766289, + "tokens_seen": 1468923904 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028028084252758274, + "loss": 2.6947, + "theoretical_loss": 3.5225729717152374, + "tokens_seen": 1468989440 + }, + { + "epoch": 0.45, + "learning_rate": 0.000280270812437312, + "loss": 2.6534, + "theoretical_loss": 3.5225590604586055, + "tokens_seen": 1469054976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802607823470411, + "loss": 2.5873, + "theoretical_loss": 3.5225451499963127, + "tokens_seen": 1469120512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028025075225677034, + "loss": 2.7467, + "theoretical_loss": 3.522531240328278, + "tokens_seen": 1469186048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028024072216649947, + "loss": 2.4201, + "theoretical_loss": 3.5225173314544205, + "tokens_seen": 1469251584 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802306920762287, + "loss": 2.6165, + "theoretical_loss": 3.5225034233746593, + "tokens_seen": 1469317120 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802206619859579, + "loss": 2.7084, + "theoretical_loss": 3.5224895160889145, + "tokens_seen": 1469382656 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028021063189568706, + "loss": 2.6769, + "theoretical_loss": 3.5224756095971044, + "tokens_seen": 1469448192 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028020060180541624, + "loss": 2.6497, + "theoretical_loss": 3.5224617038991486, + "tokens_seen": 1469513728 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801905717151454, + "loss": 2.6049, + "theoretical_loss": 3.5224477989949663, + "tokens_seen": 1469579264 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 811321, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9198386669158936, + "objective/train/theoretical_loss": 3.522433894884477, + "objective/train/tokens_used": 1490104800, + "theoretical_loss": 3.522433894884477, + "tokens_seen": 1469644800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801805416248746, + "loss": 2.7733, + "theoretical_loss": 3.522433894884477, + "tokens_seen": 1469644800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028017051153460384, + "loss": 2.8255, + "theoretical_loss": 3.5224199915675998, + "tokens_seen": 1469710336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028016048144433297, + "loss": 2.5859, + "theoretical_loss": 3.5224060890442543, + "tokens_seen": 1469775872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801504513540622, + "loss": 2.6958, + "theoretical_loss": 3.5223921873143595, + "tokens_seen": 1469841408 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028014042126379133, + "loss": 2.5037, + "theoretical_loss": 3.522378286377835, + "tokens_seen": 1469906944 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028013039117352057, + "loss": 2.6737, + "theoretical_loss": 3.5223643862346, + "tokens_seen": 1469972480 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028012036108324975, + "loss": 2.543, + "theoretical_loss": 3.522350486884574, + "tokens_seen": 1470038016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028011033099297893, + "loss": 2.5958, + "theoretical_loss": 3.522336588327676, + "tokens_seen": 1470103552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801003009027081, + "loss": 2.667, + "theoretical_loss": 3.5223226905638256, + "tokens_seen": 1470169088 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028009027081243735, + "loss": 2.7794, + "theoretical_loss": 3.522308793592943, + "tokens_seen": 1470234624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028008024072216647, + "loss": 2.7077, + "theoretical_loss": 3.5222948974149464, + "tokens_seen": 1470300160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002800702106318957, + "loss": 2.7373, + "theoretical_loss": 3.5222810020297555, + "tokens_seen": 1470365696 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028006018054162483, + "loss": 2.7383, + "theoretical_loss": 3.52226710743729, + "tokens_seen": 1470431232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028005015045135407, + "loss": 2.52, + "theoretical_loss": 3.5222532136374696, + "tokens_seen": 1470496768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028004012036108325, + "loss": 2.6957, + "theoretical_loss": 3.522239320630213, + "tokens_seen": 1470562304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028003009027081243, + "loss": 2.3979, + "theoretical_loss": 3.5222254284154406, + "tokens_seen": 1470627840 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028002006018054167, + "loss": 2.6116, + "theoretical_loss": 3.522211536993071, + "tokens_seen": 1470693376 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002800100300902708, + "loss": 2.6236, + "theoretical_loss": 3.5221976463630242, + "tokens_seen": 1470758912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028000000000000003, + "loss": 2.6138, + "theoretical_loss": 3.52218375652522, + "tokens_seen": 1470824448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799899699097292, + "loss": 2.705, + "theoretical_loss": 3.522169867479577, + "tokens_seen": 1470889984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799799398194584, + "loss": 2.7807, + "theoretical_loss": 3.5221559792260155, + "tokens_seen": 1470955520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799699097291876, + "loss": 2.619, + "theoretical_loss": 3.5221420917644544, + "tokens_seen": 1471021056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027995987963891675, + "loss": 2.547, + "theoretical_loss": 3.5221282050948144, + "tokens_seen": 1471086592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027994984954864594, + "loss": 2.7031, + "theoretical_loss": 3.5221143192170135, + "tokens_seen": 1471152128 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027993981945837517, + "loss": 2.6999, + "theoretical_loss": 3.5221004341309725, + "tokens_seen": 1471217664 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 812410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.735546588897705, + "objective/train/theoretical_loss": 3.5220865498366107, + "objective/train/tokens_used": 1491743200, + "theoretical_loss": 3.5220865498366107, + "tokens_seen": 1471283200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799297893681043, + "loss": 2.5636, + "theoretical_loss": 3.5220865498366107, + "tokens_seen": 1471283200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027991975927783353, + "loss": 2.4444, + "theoretical_loss": 3.5220726663338473, + "tokens_seen": 1471348736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799097291875627, + "loss": 2.7656, + "theoretical_loss": 3.522058783622602, + "tokens_seen": 1471414272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798996990972919, + "loss": 2.4004, + "theoretical_loss": 3.522044901702795, + "tokens_seen": 1471479808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798896690070211, + "loss": 2.4757, + "theoretical_loss": 3.5220310205743455, + "tokens_seen": 1471545344 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027987963891675026, + "loss": 2.6364, + "theoretical_loss": 3.522017140237173, + "tokens_seen": 1471610880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027986960882647944, + "loss": 2.8141, + "theoretical_loss": 3.5220032606911973, + "tokens_seen": 1471676416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798595787362087, + "loss": 2.8483, + "theoretical_loss": 3.521989381936338, + "tokens_seen": 1471741952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798495486459378, + "loss": 2.6073, + "theoretical_loss": 3.5219755039725156, + "tokens_seen": 1471807488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027983951855566704, + "loss": 2.4337, + "theoretical_loss": 3.5219616267996487, + "tokens_seen": 1471873024 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027982948846539616, + "loss": 2.6282, + "theoretical_loss": 3.5219477504176573, + "tokens_seen": 1471938560 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798194583751254, + "loss": 2.7561, + "theoretical_loss": 3.5219338748264613, + "tokens_seen": 1472004096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798094282848546, + "loss": 2.663, + "theoretical_loss": 3.5219200000259803, + "tokens_seen": 1472069632 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027979939819458376, + "loss": 2.525, + "theoretical_loss": 3.521906126016134, + "tokens_seen": 1472135168 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027978936810431294, + "loss": 2.8497, + "theoretical_loss": 3.5218922527968424, + "tokens_seen": 1472200704 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797793380140422, + "loss": 2.3666, + "theoretical_loss": 3.521878380368025, + "tokens_seen": 1472266240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797693079237713, + "loss": 2.7274, + "theoretical_loss": 3.521864508729602, + "tokens_seen": 1472331776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027975927783350054, + "loss": 2.6469, + "theoretical_loss": 3.521850637881492, + "tokens_seen": 1472397312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027974924774322967, + "loss": 2.5282, + "theoretical_loss": 3.5218367678236167, + "tokens_seen": 1472462848 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797392176529589, + "loss": 2.7885, + "theoretical_loss": 3.521822898555895, + "tokens_seen": 1472528384 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797291875626881, + "loss": 2.5835, + "theoretical_loss": 3.521809030078246, + "tokens_seen": 1472593920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027971915747241726, + "loss": 2.6981, + "theoretical_loss": 3.52179516239059, + "tokens_seen": 1472659456 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027970912738214644, + "loss": 2.6311, + "theoretical_loss": 3.521781295492847, + "tokens_seen": 1472724992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796990972918756, + "loss": 2.5466, + "theoretical_loss": 3.521767429384937, + "tokens_seen": 1472790528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796890672016048, + "loss": 2.425, + "theoretical_loss": 3.5217535640667794, + "tokens_seen": 1472856064 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 813075, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4556291103363037, + "objective/train/theoretical_loss": 3.521739699538295, + "objective/train/tokens_used": 1493381600, + "theoretical_loss": 3.521739699538295, + "tokens_seen": 1472921600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027967903711133404, + "loss": 2.5714, + "theoretical_loss": 3.521739699538295, + "tokens_seen": 1472921600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027966900702106317, + "loss": 2.6501, + "theoretical_loss": 3.521725835799403, + "tokens_seen": 1472987136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796589769307924, + "loss": 2.5784, + "theoretical_loss": 3.521711972850023, + "tokens_seen": 1473052672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027964894684052153, + "loss": 2.7543, + "theoretical_loss": 3.5216981106900755, + "tokens_seen": 1473118208 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027963891675025077, + "loss": 2.3898, + "theoretical_loss": 3.52168424931948, + "tokens_seen": 1473183744 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027962888665997995, + "loss": 2.6985, + "theoretical_loss": 3.5216703887381575, + "tokens_seen": 1473249280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027961885656970913, + "loss": 2.5342, + "theoretical_loss": 3.5216565289460267, + "tokens_seen": 1473314816 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796088264794383, + "loss": 2.4191, + "theoretical_loss": 3.5216426699430077, + "tokens_seen": 1473380352 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027959879638916755, + "loss": 2.4761, + "theoretical_loss": 3.5216288117290206, + "tokens_seen": 1473445888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027958876629889667, + "loss": 2.6122, + "theoretical_loss": 3.521614954303986, + "tokens_seen": 1473511424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795787362086259, + "loss": 2.7318, + "theoretical_loss": 3.5216010976678236, + "tokens_seen": 1473576960 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027956870611835503, + "loss": 2.7646, + "theoretical_loss": 3.521587241820453, + "tokens_seen": 1473642496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027955867602808427, + "loss": 2.5787, + "theoretical_loss": 3.521573386761794, + "tokens_seen": 1473708032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027954864593781345, + "loss": 2.6957, + "theoretical_loss": 3.521559532491768, + "tokens_seen": 1473773568 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027953861584754263, + "loss": 2.5986, + "theoretical_loss": 3.5215456790102935, + "tokens_seen": 1473839104 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795285857572718, + "loss": 2.7141, + "theoretical_loss": 3.5215318263172914, + "tokens_seen": 1473904640 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279518555667001, + "loss": 2.8128, + "theoretical_loss": 3.5215179744126814, + "tokens_seen": 1473970176 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795085255767302, + "loss": 2.8406, + "theoretical_loss": 3.521504123296384, + "tokens_seen": 1474035712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794984954864594, + "loss": 2.5559, + "theoretical_loss": 3.521490272968319, + "tokens_seen": 1474101248 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027948846539618854, + "loss": 2.5268, + "theoretical_loss": 3.521476423428407, + "tokens_seen": 1474166784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794784353059178, + "loss": 2.6032, + "theoretical_loss": 3.5214625746765673, + "tokens_seen": 1474232320 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794684052156469, + "loss": 2.6824, + "theoretical_loss": 3.5214487267127206, + "tokens_seen": 1474297856 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027945837512537614, + "loss": 2.5321, + "theoretical_loss": 3.5214348795367862, + "tokens_seen": 1474363392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794483450351053, + "loss": 2.6556, + "theoretical_loss": 3.5214210331486853, + "tokens_seen": 1474428928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794383149448345, + "loss": 2.6213, + "theoretical_loss": 3.5214071875483377, + "tokens_seen": 1474494464 + }, + { + "debugging/Self-BLEU-5": 0.4457480258618696, + "debugging/distinct-1-grams": 0.780869708352066, + "debugging/distinct-2-grams": 0.9550381307455209, + "debugging/entropy-1-grams": 5.691454067736992, + "debugging/entropy-2-grams": 6.3380960139729146, + "debugging/length": 545.0, + "debugging/num_segments": 9, + "epoch": 0.45, + "objective/train/docs_used": 814377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.860199451446533, + "objective/train/theoretical_loss": 3.5213933427356636, + "objective/train/tokens_used": 1495020000, + "theoretical_loss": 3.5213933427356636, + "tokens_seen": 1474560000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794282848545637, + "loss": 2.5364, + "theoretical_loss": 3.5213933427356636, + "tokens_seen": 1474560000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794182547642929, + "loss": 2.5116, + "theoretical_loss": 3.5213794987105826, + "tokens_seen": 1474625536 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027940822467402204, + "loss": 2.4481, + "theoretical_loss": 3.521365655473016, + "tokens_seen": 1474691072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793981945837513, + "loss": 2.3796, + "theoretical_loss": 3.5213518130228834, + "tokens_seen": 1474756608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793881644934804, + "loss": 2.6261, + "theoretical_loss": 3.5213379713601047, + "tokens_seen": 1474822144 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027937813440320964, + "loss": 2.7711, + "theoretical_loss": 3.521324130484601, + "tokens_seen": 1474887680 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793681043129388, + "loss": 2.5423, + "theoretical_loss": 3.521310290396291, + "tokens_seen": 1474953216 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279358074222668, + "loss": 2.548, + "theoretical_loss": 3.521296451095097, + "tokens_seen": 1475018752 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793480441323972, + "loss": 2.4568, + "theoretical_loss": 3.5212826125809373, + "tokens_seen": 1475084288 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027933801404212636, + "loss": 2.6257, + "theoretical_loss": 3.521268774853734, + "tokens_seen": 1475149824 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027932798395185554, + "loss": 2.7034, + "theoretical_loss": 3.521254937913406, + "tokens_seen": 1475215360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793179538615848, + "loss": 2.5689, + "theoretical_loss": 3.521241101759874, + "tokens_seen": 1475280896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793079237713139, + "loss": 2.5344, + "theoretical_loss": 3.521227266393059, + "tokens_seen": 1475346432 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027929789368104314, + "loss": 2.875, + "theoretical_loss": 3.52121343181288, + "tokens_seen": 1475411968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027928786359077227, + "loss": 2.5586, + "theoretical_loss": 3.5211995980192583, + "tokens_seen": 1475477504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792778335005015, + "loss": 2.6094, + "theoretical_loss": 3.521185765012114, + "tokens_seen": 1475543040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027926780341023074, + "loss": 2.6754, + "theoretical_loss": 3.5211719327913675, + "tokens_seen": 1475608576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027925777331995987, + "loss": 2.6065, + "theoretical_loss": 3.5211581013569395, + "tokens_seen": 1475674112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792477432296891, + "loss": 2.6373, + "theoretical_loss": 3.5211442707087497, + "tokens_seen": 1475739648 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792377131394183, + "loss": 2.6275, + "theoretical_loss": 3.521130440846719, + "tokens_seen": 1475805184 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027922768304914746, + "loss": 2.5502, + "theoretical_loss": 3.5211166117707675, + "tokens_seen": 1475870720 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027921765295887664, + "loss": 2.5877, + "theoretical_loss": 3.5211027834808157, + "tokens_seen": 1475936256 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792076228686058, + "loss": 2.6401, + "theoretical_loss": 3.5210889559767837, + "tokens_seen": 1476001792 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279197592778335, + "loss": 2.5757, + "theoretical_loss": 3.521075129258593, + "tokens_seen": 1476067328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027918756268806424, + "loss": 2.6731, + "theoretical_loss": 3.521061303326163, + "tokens_seen": 1476132864 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 814873, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.263564109802246, + "objective/train/theoretical_loss": 3.5210474781794145, + "objective/train/tokens_used": 1496658400, + "theoretical_loss": 3.5210474781794145, + "tokens_seen": 1476198400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027917753259779337, + "loss": 2.4547, + "theoretical_loss": 3.5210474781794145, + "tokens_seen": 1476198400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791675025075226, + "loss": 2.5981, + "theoretical_loss": 3.521033653818268, + "tokens_seen": 1476263936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027915747241725173, + "loss": 2.7949, + "theoretical_loss": 3.5210198302426443, + "tokens_seen": 1476329472 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027914744232698097, + "loss": 2.5497, + "theoretical_loss": 3.5210060074524634, + "tokens_seen": 1476395008 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027913741223671015, + "loss": 2.8282, + "theoretical_loss": 3.5209921854476462, + "tokens_seen": 1476460544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027912738214643933, + "loss": 2.4958, + "theoretical_loss": 3.5209783642281125, + "tokens_seen": 1476526080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791173520561685, + "loss": 2.7576, + "theoretical_loss": 3.5209645437937835, + "tokens_seen": 1476591616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791173520561685, + "loss": 2.5902, + "theoretical_loss": 3.52095072414458, + "tokens_seen": 1476657152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027910732196589775, + "loss": 2.5488, + "theoretical_loss": 3.520936905280422, + "tokens_seen": 1476722688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027909729187562687, + "loss": 2.5285, + "theoretical_loss": 3.52092308720123, + "tokens_seen": 1476788224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790872617853561, + "loss": 2.577, + "theoretical_loss": 3.520909269906925, + "tokens_seen": 1476853760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027907723169508523, + "loss": 2.5842, + "theoretical_loss": 3.520895453397427, + "tokens_seen": 1476919296 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027906720160481447, + "loss": 2.6671, + "theoretical_loss": 3.520881637672657, + "tokens_seen": 1476984832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027905717151454365, + "loss": 2.5175, + "theoretical_loss": 3.5208678227325363, + "tokens_seen": 1477050368 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027904714142427283, + "loss": 2.699, + "theoretical_loss": 3.5208540085769844, + "tokens_seen": 1477115904 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279037111334002, + "loss": 2.8946, + "theoretical_loss": 3.5208401952059223, + "tokens_seen": 1477181440 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790270812437312, + "loss": 2.6572, + "theoretical_loss": 3.5208263826192705, + "tokens_seen": 1477246976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790170511534604, + "loss": 2.5727, + "theoretical_loss": 3.5208125708169504, + "tokens_seen": 1477312512 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790070210631896, + "loss": 2.7133, + "theoretical_loss": 3.5207987597988817, + "tokens_seen": 1477378048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027899699097291874, + "loss": 2.6664, + "theoretical_loss": 3.5207849495649857, + "tokens_seen": 1477443584 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278986960882648, + "loss": 2.7478, + "theoretical_loss": 3.520771140115183, + "tokens_seen": 1477509120 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789769307923771, + "loss": 2.5194, + "theoretical_loss": 3.520757331449394, + "tokens_seen": 1477574656 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027896690070210634, + "loss": 2.5503, + "theoretical_loss": 3.52074352356754, + "tokens_seen": 1477640192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789568706118355, + "loss": 2.3763, + "theoretical_loss": 3.520729716469541, + "tokens_seen": 1477705728 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789468405215647, + "loss": 2.7193, + "theoretical_loss": 3.5207159101553183, + "tokens_seen": 1477771264 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 816004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6182334423065186, + "objective/train/theoretical_loss": 3.5207021046247924, + "objective/train/tokens_used": 1498296800, + "theoretical_loss": 3.5207021046247924, + "tokens_seen": 1477836800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789368104312939, + "loss": 2.7106, + "theoretical_loss": 3.5207021046247924, + "tokens_seen": 1477836800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789267803410231, + "loss": 2.8983, + "theoretical_loss": 3.520688299877884, + "tokens_seen": 1477902336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027891675025075224, + "loss": 2.7539, + "theoretical_loss": 3.5206744959145144, + "tokens_seen": 1477967872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789067201604815, + "loss": 2.493, + "theoretical_loss": 3.520660692734604, + "tokens_seen": 1478033408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788966900702106, + "loss": 2.607, + "theoretical_loss": 3.520646890338073, + "tokens_seen": 1478098944 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027888665997993984, + "loss": 2.7922, + "theoretical_loss": 3.5206330887248436, + "tokens_seen": 1478164480 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278876629889669, + "loss": 2.8261, + "theoretical_loss": 3.520619287894835, + "tokens_seen": 1478230016 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788665997993982, + "loss": 2.7453, + "theoretical_loss": 3.520605487847969, + "tokens_seen": 1478295552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788565697091274, + "loss": 2.5752, + "theoretical_loss": 3.520591688584167, + "tokens_seen": 1478361088 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027884653961885656, + "loss": 2.5868, + "theoretical_loss": 3.5205778901033487, + "tokens_seen": 1478426624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027883650952858574, + "loss": 2.3969, + "theoretical_loss": 3.5205640924054356, + "tokens_seen": 1478492160 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278826479438315, + "loss": 2.6064, + "theoretical_loss": 3.520550295490348, + "tokens_seen": 1478557696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788164493480441, + "loss": 2.6417, + "theoretical_loss": 3.5205364993580073, + "tokens_seen": 1478623232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027880641925777334, + "loss": 2.6554, + "theoretical_loss": 3.520522704008334, + "tokens_seen": 1478688768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027879638916750247, + "loss": 2.6519, + "theoretical_loss": 3.52050890944125, + "tokens_seen": 1478754304 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787863590772317, + "loss": 2.6336, + "theoretical_loss": 3.5204951156566753, + "tokens_seen": 1478819840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787763289869609, + "loss": 2.4454, + "theoretical_loss": 3.520481322654531, + "tokens_seen": 1478885376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027876629889669007, + "loss": 2.6013, + "theoretical_loss": 3.520467530434738, + "tokens_seen": 1478950912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027875626880641925, + "loss": 2.6672, + "theoretical_loss": 3.5204537389972175, + "tokens_seen": 1479016448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787462387161485, + "loss": 2.3893, + "theoretical_loss": 3.5204399483418904, + "tokens_seen": 1479081984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787362086258776, + "loss": 2.6592, + "theoretical_loss": 3.520426158468678, + "tokens_seen": 1479147520 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027872617853560685, + "loss": 2.6974, + "theoretical_loss": 3.5204123693775, + "tokens_seen": 1479213056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027871614844533597, + "loss": 2.4663, + "theoretical_loss": 3.520398581068279, + "tokens_seen": 1479278592 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787061183550652, + "loss": 2.5473, + "theoretical_loss": 3.520384793540935, + "tokens_seen": 1479344128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786960882647944, + "loss": 2.5937, + "theoretical_loss": 3.5203710067953895, + "tokens_seen": 1479409664 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 816537, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.965928077697754, + "objective/train/theoretical_loss": 3.5203572208315634, + "objective/train/tokens_used": 1499935200, + "theoretical_loss": 3.5203572208315634, + "tokens_seen": 1479475200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027868605817452357, + "loss": 2.6058, + "theoretical_loss": 3.5203572208315634, + "tokens_seen": 1479475200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027867602808425275, + "loss": 2.5134, + "theoretical_loss": 3.520343435649378, + "tokens_seen": 1479540736 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027866599799398193, + "loss": 2.7719, + "theoretical_loss": 3.520329651248754, + "tokens_seen": 1479606272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786559679037111, + "loss": 2.7405, + "theoretical_loss": 3.5203158676296127, + "tokens_seen": 1479671808 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027864593781344035, + "loss": 2.5804, + "theoretical_loss": 3.520302084791875, + "tokens_seen": 1479737344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786359077231695, + "loss": 2.5212, + "theoretical_loss": 3.5202883027354614, + "tokens_seen": 1479802880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786258776328987, + "loss": 2.7162, + "theoretical_loss": 3.5202745214602946, + "tokens_seen": 1479868416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786158475426279, + "loss": 2.5461, + "theoretical_loss": 3.5202607409662945, + "tokens_seen": 1479933952 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027860581745235707, + "loss": 2.6608, + "theoretical_loss": 3.520246961253383, + "tokens_seen": 1479999488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027859578736208625, + "loss": 2.5891, + "theoretical_loss": 3.52023318232148, + "tokens_seen": 1480065024 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027858575727181544, + "loss": 2.6021, + "theoretical_loss": 3.5202194041705077, + "tokens_seen": 1480130560 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785757271815446, + "loss": 2.6964, + "theoretical_loss": 3.5202056268003874, + "tokens_seen": 1480196096 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027856569709127385, + "loss": 2.9855, + "theoretical_loss": 3.520191850211039, + "tokens_seen": 1480261632 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278555667001003, + "loss": 2.6498, + "theoretical_loss": 3.520178074402385, + "tokens_seen": 1480327168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785456369107322, + "loss": 2.5365, + "theoretical_loss": 3.520164299374347, + "tokens_seen": 1480392704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027853560682046134, + "loss": 2.6188, + "theoretical_loss": 3.5201505251268443, + "tokens_seen": 1480458240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785255767301906, + "loss": 2.638, + "theoretical_loss": 3.5201367516597992, + "tokens_seen": 1480523776 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785155466399198, + "loss": 2.7699, + "theoretical_loss": 3.520122978973134, + "tokens_seen": 1480589312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027850551654964894, + "loss": 2.7147, + "theoretical_loss": 3.520109207066768, + "tokens_seen": 1480654848 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784954864593782, + "loss": 2.5845, + "theoretical_loss": 3.5200954359406236, + "tokens_seen": 1480720384 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784854563691073, + "loss": 2.6269, + "theoretical_loss": 3.520081665594622, + "tokens_seen": 1480785920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027847542627883654, + "loss": 2.4266, + "theoretical_loss": 3.5200678960286838, + "tokens_seen": 1480851456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784653961885657, + "loss": 2.4685, + "theoretical_loss": 3.520054127242731, + "tokens_seen": 1480916992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784553660982949, + "loss": 2.7099, + "theoretical_loss": 3.5200403592366847, + "tokens_seen": 1480982528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784453360080241, + "loss": 2.3486, + "theoretical_loss": 3.520026592010466, + "tokens_seen": 1481048064 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 817599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7796854972839355, + "objective/train/theoretical_loss": 3.520012825563997, + "objective/train/tokens_used": 1501573600, + "theoretical_loss": 3.520012825563997, + "tokens_seen": 1481113600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784353059177533, + "loss": 2.6445, + "theoretical_loss": 3.520012825563997, + "tokens_seen": 1481113600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027842527582748244, + "loss": 2.7275, + "theoretical_loss": 3.5199990598971977, + "tokens_seen": 1481179136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784152457372117, + "loss": 2.559, + "theoretical_loss": 3.519985295009991, + "tokens_seen": 1481244672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784052156469408, + "loss": 2.6555, + "theoretical_loss": 3.519971530902297, + "tokens_seen": 1481310208 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027839518555667004, + "loss": 2.6921, + "theoretical_loss": 3.519957767574038, + "tokens_seen": 1481375744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783851554663992, + "loss": 2.8146, + "theoretical_loss": 3.5199440050251343, + "tokens_seen": 1481441280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783751253761284, + "loss": 2.5935, + "theoretical_loss": 3.519930243255508, + "tokens_seen": 1481506816 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783650952858576, + "loss": 2.5259, + "theoretical_loss": 3.5199164822650806, + "tokens_seen": 1481572352 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027835506519558676, + "loss": 2.454, + "theoretical_loss": 3.5199027220537737, + "tokens_seen": 1481637888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027834503510531594, + "loss": 2.717, + "theoretical_loss": 3.5198889626215077, + "tokens_seen": 1481703424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783350050150452, + "loss": 2.5306, + "theoretical_loss": 3.5198752039682057, + "tokens_seen": 1481768960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783249749247743, + "loss": 2.5441, + "theoretical_loss": 3.5198614460937874, + "tokens_seen": 1481834496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027831494483450354, + "loss": 2.5472, + "theoretical_loss": 3.5198476889981754, + "tokens_seen": 1481900032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027830491474423267, + "loss": 2.6856, + "theoretical_loss": 3.51983393268129, + "tokens_seen": 1481965568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782948846539619, + "loss": 2.4332, + "theoretical_loss": 3.519820177143054, + "tokens_seen": 1482031104 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782848545636911, + "loss": 2.7552, + "theoretical_loss": 3.5198064223833887, + "tokens_seen": 1482096640 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027827482447342027, + "loss": 2.5342, + "theoretical_loss": 3.519792668402215, + "tokens_seen": 1482162176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027826479438314945, + "loss": 2.5159, + "theoretical_loss": 3.5197789151994545, + "tokens_seen": 1482227712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782547642928787, + "loss": 2.5585, + "theoretical_loss": 3.5197651627750295, + "tokens_seen": 1482293248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782447342026078, + "loss": 2.5253, + "theoretical_loss": 3.5197514111288606, + "tokens_seen": 1482358784 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027823470411233705, + "loss": 2.6442, + "theoretical_loss": 3.5197376602608696, + "tokens_seen": 1482424320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027822467402206617, + "loss": 2.7405, + "theoretical_loss": 3.5197239101709785, + "tokens_seen": 1482489856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782146439317954, + "loss": 2.7626, + "theoretical_loss": 3.5197101608591086, + "tokens_seen": 1482555392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782046138415246, + "loss": 2.4868, + "theoretical_loss": 3.5196964123251813, + "tokens_seen": 1482620928 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027819458375125377, + "loss": 2.5604, + "theoretical_loss": 3.5196826645691184, + "tokens_seen": 1482686464 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 818835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.735487937927246, + "objective/train/theoretical_loss": 3.5196689175908418, + "objective/train/tokens_used": 1503212000, + "theoretical_loss": 3.5196689175908418, + "tokens_seen": 1482752000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027818455366098295, + "loss": 2.6771, + "theoretical_loss": 3.5196689175908418, + "tokens_seen": 1482752000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027817452357071213, + "loss": 2.4758, + "theoretical_loss": 3.5196551713902724, + "tokens_seen": 1482817536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781644934804413, + "loss": 2.6044, + "theoretical_loss": 3.5196414259673325, + "tokens_seen": 1482883072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027815446339017055, + "loss": 2.6185, + "theoretical_loss": 3.519627681321943, + "tokens_seen": 1482948608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781444332998997, + "loss": 2.6867, + "theoretical_loss": 3.519613937454026, + "tokens_seen": 1483014144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781344032096289, + "loss": 2.7563, + "theoretical_loss": 3.5196001943635036, + "tokens_seen": 1483079680 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781243731193581, + "loss": 2.7083, + "theoretical_loss": 3.5195864520502975, + "tokens_seen": 1483145216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781143430290873, + "loss": 2.4394, + "theoretical_loss": 3.5195727105143284, + "tokens_seen": 1483210752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027810431293881645, + "loss": 2.5391, + "theoretical_loss": 3.5195589697555185, + "tokens_seen": 1483276288 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027809428284854564, + "loss": 2.72, + "theoretical_loss": 3.51954522977379, + "tokens_seen": 1483341824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780842527582748, + "loss": 2.7018, + "theoretical_loss": 3.5195314905690642, + "tokens_seen": 1483407360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027807422266800405, + "loss": 2.66, + "theoretical_loss": 3.519517752141262, + "tokens_seen": 1483472896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780641925777332, + "loss": 2.6427, + "theoretical_loss": 3.519504014490307, + "tokens_seen": 1483538432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780541624874624, + "loss": 2.7975, + "theoretical_loss": 3.5194902776161197, + "tokens_seen": 1483603968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027804413239719154, + "loss": 2.6736, + "theoretical_loss": 3.519476541518622, + "tokens_seen": 1483669504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780341023069208, + "loss": 2.5769, + "theoretical_loss": 3.5194628061977355, + "tokens_seen": 1483735040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027802407221664996, + "loss": 2.748, + "theoretical_loss": 3.5194490716533826, + "tokens_seen": 1483800576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027801404212637914, + "loss": 2.4636, + "theoretical_loss": 3.5194353378854846, + "tokens_seen": 1483866112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780040120361083, + "loss": 2.6612, + "theoretical_loss": 3.5194216048939637, + "tokens_seen": 1483931648 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779939819458375, + "loss": 2.4891, + "theoretical_loss": 3.519407872678742, + "tokens_seen": 1483997184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779839518555667, + "loss": 2.8148, + "theoretical_loss": 3.51939414123974, + "tokens_seen": 1484062720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779739217652959, + "loss": 2.8085, + "theoretical_loss": 3.519380410576881, + "tokens_seen": 1484128256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027796389167502504, + "loss": 2.6915, + "theoretical_loss": 3.5193666806900863, + "tokens_seen": 1484193792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779538615847543, + "loss": 2.613, + "theoretical_loss": 3.519352951579277, + "tokens_seen": 1484259328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027794383149448346, + "loss": 2.4977, + "theoretical_loss": 3.5193392232443763, + "tokens_seen": 1484324864 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 819535, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2024571895599365, + "objective/train/theoretical_loss": 3.519325495685306, + "objective/train/tokens_used": 1504850400, + "theoretical_loss": 3.519325495685306, + "tokens_seen": 1484390400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027793380140421264, + "loss": 2.6604, + "theoretical_loss": 3.519325495685306, + "tokens_seen": 1484390400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779237713139418, + "loss": 2.6439, + "theoretical_loss": 3.519311768901987, + "tokens_seen": 1484455936 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277913741223671, + "loss": 2.5186, + "theoretical_loss": 3.5192980428943414, + "tokens_seen": 1484521472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779037111334002, + "loss": 2.6832, + "theoretical_loss": 3.5192843176622923, + "tokens_seen": 1484587008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778936810431294, + "loss": 2.6472, + "theoretical_loss": 3.5192705932057606, + "tokens_seen": 1484652544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027788365095285855, + "loss": 2.66, + "theoretical_loss": 3.5192568695246678, + "tokens_seen": 1484718080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778736208625878, + "loss": 2.6594, + "theoretical_loss": 3.5192431466189373, + "tokens_seen": 1484783616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778635907723169, + "loss": 2.629, + "theoretical_loss": 3.51922942448849, + "tokens_seen": 1484849152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027785356068204614, + "loss": 2.7417, + "theoretical_loss": 3.5192157031332485, + "tokens_seen": 1484914688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778435305917753, + "loss": 2.7664, + "theoretical_loss": 3.519201982553134, + "tokens_seen": 1484980224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778335005015045, + "loss": 2.6715, + "theoretical_loss": 3.5191882627480693, + "tokens_seen": 1485045760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778234704112337, + "loss": 2.5785, + "theoretical_loss": 3.519174543717976, + "tokens_seen": 1485111296 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027781344032096287, + "loss": 2.6659, + "theoretical_loss": 3.5191608254627766, + "tokens_seen": 1485176832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027780341023069205, + "loss": 2.5638, + "theoretical_loss": 3.519147107982392, + "tokens_seen": 1485242368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777933801404213, + "loss": 2.8271, + "theoretical_loss": 3.5191333912767457, + "tokens_seen": 1485307904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777833500501504, + "loss": 2.8528, + "theoretical_loss": 3.5191196753457588, + "tokens_seen": 1485373440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027777331995987965, + "loss": 2.5073, + "theoretical_loss": 3.519105960189354, + "tokens_seen": 1485438976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777632898696089, + "loss": 2.5813, + "theoretical_loss": 3.5190922458074527, + "tokens_seen": 1485504512 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277753259779338, + "loss": 2.5409, + "theoretical_loss": 3.519078532199978, + "tokens_seen": 1485570048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027774322968906725, + "loss": 2.7484, + "theoretical_loss": 3.51906481936685, + "tokens_seen": 1485635584 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027773319959879637, + "loss": 2.4342, + "theoretical_loss": 3.5190511073079938, + "tokens_seen": 1485701120 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777231695085256, + "loss": 2.5802, + "theoretical_loss": 3.5190373960233288, + "tokens_seen": 1485766656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777131394182548, + "loss": 2.5896, + "theoretical_loss": 3.5190236855127788, + "tokens_seen": 1485832192 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027770310932798397, + "loss": 2.7993, + "theoretical_loss": 3.519009975776265, + "tokens_seen": 1485897728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027769307923771315, + "loss": 2.6962, + "theoretical_loss": 3.5189962668137102, + "tokens_seen": 1485963264 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 820839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.42441987991333, + "objective/train/theoretical_loss": 3.5189825586250363, + "objective/train/tokens_used": 1506488800, + "theoretical_loss": 3.5189825586250363, + "tokens_seen": 1486028800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027768304914744233, + "loss": 2.7172, + "theoretical_loss": 3.5189825586250363, + "tokens_seen": 1486028800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776730190571715, + "loss": 2.5732, + "theoretical_loss": 3.5189688512101656, + "tokens_seen": 1486094336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027766298896690075, + "loss": 2.4703, + "theoretical_loss": 3.518955144569021, + "tokens_seen": 1486159872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776529588766299, + "loss": 2.7413, + "theoretical_loss": 3.518941438701523, + "tokens_seen": 1486225408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776429287863591, + "loss": 2.5478, + "theoretical_loss": 3.518927733607595, + "tokens_seen": 1486290944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776328986960883, + "loss": 2.7123, + "theoretical_loss": 3.518914029287159, + "tokens_seen": 1486356480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776228686058175, + "loss": 2.6896, + "theoretical_loss": 3.518900325740137, + "tokens_seen": 1486422016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027761283851554665, + "loss": 2.7992, + "theoretical_loss": 3.518886622966452, + "tokens_seen": 1486487552 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027760280842527584, + "loss": 2.6367, + "theoretical_loss": 3.518872920966025, + "tokens_seen": 1486553088 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277592778335005, + "loss": 2.8035, + "theoretical_loss": 3.51885921973878, + "tokens_seen": 1486618624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027758274824473425, + "loss": 2.6149, + "theoretical_loss": 3.5188455192846377, + "tokens_seen": 1486684160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775727181544634, + "loss": 2.771, + "theoretical_loss": 3.5188318196035215, + "tokens_seen": 1486749696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775626880641926, + "loss": 2.9162, + "theoretical_loss": 3.518818120695353, + "tokens_seen": 1486815232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027755265797392174, + "loss": 2.8375, + "theoretical_loss": 3.5188044225600548, + "tokens_seen": 1486880768 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277542627883651, + "loss": 2.7337, + "theoretical_loss": 3.518790725197549, + "tokens_seen": 1486946304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027753259779338016, + "loss": 2.836, + "theoretical_loss": 3.5187770286077584, + "tokens_seen": 1487011840 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027752256770310934, + "loss": 2.7107, + "theoretical_loss": 3.5187633327906047, + "tokens_seen": 1487077376 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775125376128385, + "loss": 2.5469, + "theoretical_loss": 3.5187496377460112, + "tokens_seen": 1487142912 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775025075225677, + "loss": 2.6512, + "theoretical_loss": 3.5187359434738994, + "tokens_seen": 1487208448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774924774322969, + "loss": 2.7829, + "theoretical_loss": 3.5187222499741924, + "tokens_seen": 1487273984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774824473420261, + "loss": 2.567, + "theoretical_loss": 3.518708557246812, + "tokens_seen": 1487339520 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027747241725175524, + "loss": 2.5233, + "theoretical_loss": 3.518694865291681, + "tokens_seen": 1487405056 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774623871614845, + "loss": 2.6127, + "theoretical_loss": 3.518681174108721, + "tokens_seen": 1487470592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027745235707121366, + "loss": 2.5781, + "theoretical_loss": 3.518667483697856, + "tokens_seen": 1487536128 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027744232698094284, + "loss": 2.6465, + "theoretical_loss": 3.5186537940590075, + "tokens_seen": 1487601664 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 821509, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.194403648376465, + "objective/train/theoretical_loss": 3.5186401051920977, + "objective/train/tokens_used": 1508127200, + "theoretical_loss": 3.5186401051920977, + "tokens_seen": 1487667200 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277432296890672, + "loss": 2.4996, + "theoretical_loss": 3.5186401051920977, + "tokens_seen": 1487667200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774222668004012, + "loss": 2.4979, + "theoretical_loss": 3.5186264170970496, + "tokens_seen": 1487732736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774122367101304, + "loss": 2.7645, + "theoretical_loss": 3.518612729773785, + "tokens_seen": 1487798272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774022066198596, + "loss": 2.5919, + "theoretical_loss": 3.5185990432222267, + "tokens_seen": 1487863808 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027739217652958875, + "loss": 2.7519, + "theoretical_loss": 3.5185853574422983, + "tokens_seen": 1487929344 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277382146439318, + "loss": 2.633, + "theoretical_loss": 3.5185716724339207, + "tokens_seen": 1487994880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773721163490471, + "loss": 2.5317, + "theoretical_loss": 3.518557988197017, + "tokens_seen": 1488060416 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027736208625877634, + "loss": 2.6653, + "theoretical_loss": 3.5185443047315106, + "tokens_seen": 1488125952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773520561685055, + "loss": 2.6096, + "theoretical_loss": 3.5185306220373223, + "tokens_seen": 1488191488 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773420260782347, + "loss": 2.7613, + "theoretical_loss": 3.5185169401143765, + "tokens_seen": 1488257024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773319959879639, + "loss": 2.7551, + "theoretical_loss": 3.5185032589625944, + "tokens_seen": 1488322560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027732196589769307, + "loss": 2.6489, + "theoretical_loss": 3.518489578581899, + "tokens_seen": 1488388096 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027731193580742225, + "loss": 2.6812, + "theoretical_loss": 3.518475898972213, + "tokens_seen": 1488453632 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773019057171515, + "loss": 2.6617, + "theoretical_loss": 3.5184622201334594, + "tokens_seen": 1488519168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772918756268806, + "loss": 2.5584, + "theoretical_loss": 3.51844854206556, + "tokens_seen": 1488584704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027728184553660985, + "loss": 2.6746, + "theoretical_loss": 3.5184348647684383, + "tokens_seen": 1488650240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027727181544633903, + "loss": 2.698, + "theoretical_loss": 3.5184211882420158, + "tokens_seen": 1488715776 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772617853560682, + "loss": 2.8228, + "theoretical_loss": 3.518407512486216, + "tokens_seen": 1488781312 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772517552657974, + "loss": 2.5921, + "theoretical_loss": 3.5183938375009616, + "tokens_seen": 1488846848 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027724172517552657, + "loss": 2.7007, + "theoretical_loss": 3.5183801632861744, + "tokens_seen": 1488912384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027723169508525575, + "loss": 2.8786, + "theoretical_loss": 3.5183664898417786, + "tokens_seen": 1488977920 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277221664994985, + "loss": 2.8498, + "theoretical_loss": 3.5183528171676954, + "tokens_seen": 1489043456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772116349047141, + "loss": 2.6078, + "theoretical_loss": 3.518339145263848, + "tokens_seen": 1489108992 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027720160481444335, + "loss": 2.633, + "theoretical_loss": 3.51832547413016, + "tokens_seen": 1489174528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771915747241725, + "loss": 2.8041, + "theoretical_loss": 3.5183118037665526, + "tokens_seen": 1489240064 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 822598, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.609518527984619, + "objective/train/theoretical_loss": 3.5182981341729493, + "objective/train/tokens_used": 1509765600, + "theoretical_loss": 3.5182981341729493, + "tokens_seen": 1489305600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771815446339017, + "loss": 2.5348, + "theoretical_loss": 3.5182981341729493, + "tokens_seen": 1489305600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771715145436309, + "loss": 2.7176, + "theoretical_loss": 3.518284465349273, + "tokens_seen": 1489371136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771614844533601, + "loss": 2.5294, + "theoretical_loss": 3.5182707972954463, + "tokens_seen": 1489436672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027715145436308926, + "loss": 2.8534, + "theoretical_loss": 3.518257130011392, + "tokens_seen": 1489502208 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771414242728185, + "loss": 2.445, + "theoretical_loss": 3.5182434634970328, + "tokens_seen": 1489567744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771313941825476, + "loss": 2.6848, + "theoretical_loss": 3.5182297977522916, + "tokens_seen": 1489633280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027712136409227685, + "loss": 2.7385, + "theoretical_loss": 3.518216132777091, + "tokens_seen": 1489698816 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277111334002006, + "loss": 2.5515, + "theoretical_loss": 3.5182024685713538, + "tokens_seen": 1489764352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771013039117352, + "loss": 2.5135, + "theoretical_loss": 3.518188805135003, + "tokens_seen": 1489829888 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770912738214644, + "loss": 2.7728, + "theoretical_loss": 3.5181751424679613, + "tokens_seen": 1489895424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770812437311936, + "loss": 2.7078, + "theoretical_loss": 3.518161480570152, + "tokens_seen": 1489960960 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027707121364092276, + "loss": 2.5694, + "theoretical_loss": 3.5181478194414977, + "tokens_seen": 1490026496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027706118355065194, + "loss": 2.5484, + "theoretical_loss": 3.518134159081921, + "tokens_seen": 1490092032 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770511534603811, + "loss": 2.5302, + "theoretical_loss": 3.518120499491345, + "tokens_seen": 1490157568 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027704112337011036, + "loss": 2.7996, + "theoretical_loss": 3.5181068406696925, + "tokens_seen": 1490223104 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770310932798395, + "loss": 2.6767, + "theoretical_loss": 3.5180931826168864, + "tokens_seen": 1490288640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770210631895687, + "loss": 2.6392, + "theoretical_loss": 3.51807952533285, + "tokens_seen": 1490354176 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770110330992979, + "loss": 2.8262, + "theoretical_loss": 3.518065868817505, + "tokens_seen": 1490419712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770010030090271, + "loss": 2.6533, + "theoretical_loss": 3.518052213070776, + "tokens_seen": 1490485248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769909729187563, + "loss": 2.5859, + "theoretical_loss": 3.5180385580925853, + "tokens_seen": 1490550784 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027698094282848544, + "loss": 2.3949, + "theoretical_loss": 3.5180249038828553, + "tokens_seen": 1490616320 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769709127382147, + "loss": 2.6479, + "theoretical_loss": 3.5180112504415098, + "tokens_seen": 1490681856 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027696088264794386, + "loss": 2.57, + "theoretical_loss": 3.5179975977684714, + "tokens_seen": 1490747392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027695085255767304, + "loss": 2.6443, + "theoretical_loss": 3.517983945863663, + "tokens_seen": 1490812928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769408224674022, + "loss": 2.4459, + "theoretical_loss": 3.5179702947270073, + "tokens_seen": 1490878464 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 823992, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9485185146331787, + "objective/train/theoretical_loss": 3.517956644358428, + "objective/train/tokens_used": 1511404000, + "theoretical_loss": 3.517956644358428, + "tokens_seen": 1490944000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769307923771314, + "loss": 2.7354, + "theoretical_loss": 3.517956644358428, + "tokens_seen": 1490944000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769207622868606, + "loss": 2.6115, + "theoretical_loss": 3.517942994757848, + "tokens_seen": 1491009536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769107321965898, + "loss": 2.6366, + "theoretical_loss": 3.51792934592519, + "tokens_seen": 1491075072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027690070210631895, + "loss": 2.5609, + "theoretical_loss": 3.517915697860377, + "tokens_seen": 1491140608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768906720160482, + "loss": 2.812, + "theoretical_loss": 3.5179020505633325, + "tokens_seen": 1491206144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768806419257773, + "loss": 2.6548, + "theoretical_loss": 3.5178884040339797, + "tokens_seen": 1491271680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027687061183550655, + "loss": 2.7482, + "theoretical_loss": 3.517874758272241, + "tokens_seen": 1491337216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768605817452357, + "loss": 2.458, + "theoretical_loss": 3.5178611132780393, + "tokens_seen": 1491402752 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768505516549649, + "loss": 2.5753, + "theoretical_loss": 3.517847469051299, + "tokens_seen": 1491468288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768405215646941, + "loss": 2.5678, + "theoretical_loss": 3.517833825591942, + "tokens_seen": 1491533824 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027683049147442327, + "loss": 2.6824, + "theoretical_loss": 3.517820182899892, + "tokens_seen": 1491599360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027682046138415245, + "loss": 2.5631, + "theoretical_loss": 3.517806540975072, + "tokens_seen": 1491664896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768104312938817, + "loss": 2.7715, + "theoretical_loss": 3.5177928998174055, + "tokens_seen": 1491730432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768004012036108, + "loss": 2.5204, + "theoretical_loss": 3.517779259426815, + "tokens_seen": 1491795968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027679037111334005, + "loss": 2.8217, + "theoretical_loss": 3.5177656198032237, + "tokens_seen": 1491861504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027678034102306923, + "loss": 2.7824, + "theoretical_loss": 3.5177519809465556, + "tokens_seen": 1491927040 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767703109327984, + "loss": 2.7031, + "theoretical_loss": 3.517738342856733, + "tokens_seen": 1491992576 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767602808425276, + "loss": 2.5397, + "theoretical_loss": 3.5177247055336798, + "tokens_seen": 1492058112 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027675025075225677, + "loss": 2.4636, + "theoretical_loss": 3.5177110689773183, + "tokens_seen": 1492123648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027674022066198595, + "loss": 2.6908, + "theoretical_loss": 3.5176974331875726, + "tokens_seen": 1492189184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767301905717152, + "loss": 2.4168, + "theoretical_loss": 3.5176837981643656, + "tokens_seen": 1492254720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767201604814443, + "loss": 2.7574, + "theoretical_loss": 3.5176701639076207, + "tokens_seen": 1492320256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027671013039117355, + "loss": 2.7032, + "theoretical_loss": 3.517656530417261, + "tokens_seen": 1492385792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767001003009027, + "loss": 2.5744, + "theoretical_loss": 3.5176428976932095, + "tokens_seen": 1492451328 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766900702106319, + "loss": 2.6954, + "theoretical_loss": 3.51762926573539, + "tokens_seen": 1492516864 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 824636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.556931734085083, + "objective/train/theoretical_loss": 3.5176156345437253, + "objective/train/tokens_used": 1513042400, + "theoretical_loss": 3.5176156345437253, + "tokens_seen": 1492582400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766800401203611, + "loss": 2.7401, + "theoretical_loss": 3.5176156345437253, + "tokens_seen": 1492582400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766700100300903, + "loss": 2.4808, + "theoretical_loss": 3.5176020041181397, + "tokens_seen": 1492647936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027665997993981946, + "loss": 2.4619, + "theoretical_loss": 3.5175883744585548, + "tokens_seen": 1492713472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766499498495487, + "loss": 2.6713, + "theoretical_loss": 3.517574745564896, + "tokens_seen": 1492779008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766399197592778, + "loss": 2.7158, + "theoretical_loss": 3.5175611174370847, + "tokens_seen": 1492844544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027662988966900705, + "loss": 2.6191, + "theoretical_loss": 3.5175474900750454, + "tokens_seen": 1492910080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766198595787362, + "loss": 2.637, + "theoretical_loss": 3.5175338634787012, + "tokens_seen": 1492975616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766098294884654, + "loss": 2.5831, + "theoretical_loss": 3.5175202376479753, + "tokens_seen": 1493041152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765997993981946, + "loss": 2.5333, + "theoretical_loss": 3.517506612582791, + "tokens_seen": 1493106688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765897693079238, + "loss": 2.6254, + "theoretical_loss": 3.5174929882830726, + "tokens_seen": 1493172224 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027657973921765296, + "loss": 2.6903, + "theoretical_loss": 3.517479364748742, + "tokens_seen": 1493237760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027656970912738214, + "loss": 2.5451, + "theoretical_loss": 3.517465741979724, + "tokens_seen": 1493303296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765596790371113, + "loss": 2.3796, + "theoretical_loss": 3.517452119975941, + "tokens_seen": 1493368832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027654964894684056, + "loss": 2.4264, + "theoretical_loss": 3.5174384987373175, + "tokens_seen": 1493434368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765396188565697, + "loss": 2.4714, + "theoretical_loss": 3.5174248782637756, + "tokens_seen": 1493499904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765295887662989, + "loss": 2.5993, + "theoretical_loss": 3.51741125855524, + "tokens_seen": 1493565440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027651955867602805, + "loss": 2.6736, + "theoretical_loss": 3.5173976396116333, + "tokens_seen": 1493630976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765095285857573, + "loss": 2.6955, + "theoretical_loss": 3.5173840214328793, + "tokens_seen": 1493696512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027649949849548646, + "loss": 2.5953, + "theoretical_loss": 3.5173704040189016, + "tokens_seen": 1493762048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027648946840521564, + "loss": 2.7007, + "theoretical_loss": 3.517356787369624, + "tokens_seen": 1493827584 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764794383149448, + "loss": 2.6716, + "theoretical_loss": 3.517343171484969, + "tokens_seen": 1493893120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027646940822467406, + "loss": 2.5113, + "theoretical_loss": 3.517329556364861, + "tokens_seen": 1493958656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764593781344032, + "loss": 2.4526, + "theoretical_loss": 3.5173159420092235, + "tokens_seen": 1494024192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764493480441324, + "loss": 2.4199, + "theoretical_loss": 3.51730232841798, + "tokens_seen": 1494089728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027643931795386155, + "loss": 2.7458, + "theoretical_loss": 3.517288715591053, + "tokens_seen": 1494155264 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 825478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.548858642578125, + "objective/train/theoretical_loss": 3.517275103528368, + "objective/train/tokens_used": 1514680800, + "theoretical_loss": 3.517275103528368, + "tokens_seen": 1494220800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764292878635908, + "loss": 2.8416, + "theoretical_loss": 3.517275103528368, + "tokens_seen": 1494220800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027641925777331997, + "loss": 2.7125, + "theoretical_loss": 3.5172614922298466, + "tokens_seen": 1494286336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027640922768304915, + "loss": 2.6171, + "theoretical_loss": 3.517247881695414, + "tokens_seen": 1494351872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027639919759277833, + "loss": 2.3508, + "theoretical_loss": 3.5172342719249925, + "tokens_seen": 1494417408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763891675025075, + "loss": 2.549, + "theoretical_loss": 3.5172206629185068, + "tokens_seen": 1494482944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763791374122367, + "loss": 2.6508, + "theoretical_loss": 3.51720705467588, + "tokens_seen": 1494548480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763691073219659, + "loss": 2.685, + "theoretical_loss": 3.5171934471970356, + "tokens_seen": 1494614016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027635907723169505, + "loss": 2.7894, + "theoretical_loss": 3.517179840481897, + "tokens_seen": 1494679552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763490471414243, + "loss": 2.5345, + "theoretical_loss": 3.517166234530389, + "tokens_seen": 1494745088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763390170511534, + "loss": 2.8126, + "theoretical_loss": 3.5171526293424344, + "tokens_seen": 1494810624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027632898696088265, + "loss": 2.6083, + "theoretical_loss": 3.517139024917957, + "tokens_seen": 1494876160 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027631895687061183, + "loss": 2.343, + "theoretical_loss": 3.51712542125688, + "tokens_seen": 1494941696 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276308926780341, + "loss": 2.7823, + "theoretical_loss": 3.5171118183591283, + "tokens_seen": 1495007232 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762988966900702, + "loss": 2.5965, + "theoretical_loss": 3.5170982162246247, + "tokens_seen": 1495072768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027628886659979943, + "loss": 2.634, + "theoretical_loss": 3.5170846148532933, + "tokens_seen": 1495138304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027627883650952856, + "loss": 2.7194, + "theoretical_loss": 3.517071014245057, + "tokens_seen": 1495203840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762688064192578, + "loss": 2.3761, + "theoretical_loss": 3.517057414399841, + "tokens_seen": 1495269376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027625877632898697, + "loss": 2.5364, + "theoretical_loss": 3.5170438153175683, + "tokens_seen": 1495334912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027624874623871615, + "loss": 2.6406, + "theoretical_loss": 3.5170302169981618, + "tokens_seen": 1495400448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762387161484454, + "loss": 2.6997, + "theoretical_loss": 3.517016619441547, + "tokens_seen": 1495465984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762286860581745, + "loss": 2.6544, + "theoretical_loss": 3.5170030226476463, + "tokens_seen": 1495531520 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027621865596790375, + "loss": 2.7054, + "theoretical_loss": 3.5169894266163837, + "tokens_seen": 1495597056 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762086258776329, + "loss": 2.4439, + "theoretical_loss": 3.516975831347684, + "tokens_seen": 1495662592 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761985957873621, + "loss": 2.5708, + "theoretical_loss": 3.51696223684147, + "tokens_seen": 1495728128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761885656970913, + "loss": 2.494, + "theoretical_loss": 3.516948643097666, + "tokens_seen": 1495793664 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 826035, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.259209156036377, + "objective/train/theoretical_loss": 3.5169350501161953, + "objective/train/tokens_used": 1516319200, + "theoretical_loss": 3.5169350501161953, + "tokens_seen": 1495859200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761785356068205, + "loss": 2.5004, + "theoretical_loss": 3.5169350501161953, + "tokens_seen": 1495859200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027616850551654966, + "loss": 2.532, + "theoretical_loss": 3.5169214578969825, + "tokens_seen": 1495924736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761584754262789, + "loss": 2.5712, + "theoretical_loss": 3.5169078664399507, + "tokens_seen": 1495990272 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276148445336008, + "loss": 2.8897, + "theoretical_loss": 3.516894275745025, + "tokens_seen": 1496055808 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027613841524573725, + "loss": 2.7066, + "theoretical_loss": 3.5168806858121275, + "tokens_seen": 1496121344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761283851554664, + "loss": 2.7881, + "theoretical_loss": 3.5168670966411835, + "tokens_seen": 1496186880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761183550651956, + "loss": 2.7021, + "theoretical_loss": 3.5168535082321166, + "tokens_seen": 1496252416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761083249749248, + "loss": 2.7848, + "theoretical_loss": 3.5168399205848506, + "tokens_seen": 1496317952 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276098294884654, + "loss": 2.4479, + "theoretical_loss": 3.516826333699309, + "tokens_seen": 1496383488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027608826479438316, + "loss": 2.4746, + "theoretical_loss": 3.5168127475754165, + "tokens_seen": 1496449024 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027607823470411234, + "loss": 2.8988, + "theoretical_loss": 3.516799162213097, + "tokens_seen": 1496514560 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760682046138415, + "loss": 2.6517, + "theoretical_loss": 3.5167855776122736, + "tokens_seen": 1496580096 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027605817452357076, + "loss": 2.6976, + "theoretical_loss": 3.516771993772871, + "tokens_seen": 1496645632 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760481444332999, + "loss": 2.6669, + "theoretical_loss": 3.516758410694813, + "tokens_seen": 1496711168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760381143430291, + "loss": 2.7465, + "theoretical_loss": 3.516744828378024, + "tokens_seen": 1496776704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027602808425275825, + "loss": 2.6385, + "theoretical_loss": 3.5167312468224274, + "tokens_seen": 1496842240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760180541624875, + "loss": 2.703, + "theoretical_loss": 3.5167176660279473, + "tokens_seen": 1496907776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027600802407221666, + "loss": 2.8551, + "theoretical_loss": 3.516704085994508, + "tokens_seen": 1496973312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027599799398194584, + "loss": 2.8573, + "theoretical_loss": 3.516690506722033, + "tokens_seen": 1497038848 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275987963891675, + "loss": 2.6318, + "theoretical_loss": 3.5166769282104475, + "tokens_seen": 1497104384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027597793380140426, + "loss": 2.7615, + "theoretical_loss": 3.516663350459674, + "tokens_seen": 1497169920 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759679037111334, + "loss": 2.6212, + "theoretical_loss": 3.516649773469638, + "tokens_seen": 1497235456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759578736208626, + "loss": 2.8382, + "theoretical_loss": 3.5166361972402624, + "tokens_seen": 1497300992 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027594784353059175, + "loss": 2.5502, + "theoretical_loss": 3.5166226217714724, + "tokens_seen": 1497366528 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275937813440321, + "loss": 2.6544, + "theoretical_loss": 3.5166090470631914, + "tokens_seen": 1497432064 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 827284, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4943959712982178, + "objective/train/theoretical_loss": 3.516595473115344, + "objective/train/tokens_used": 1517957600, + "theoretical_loss": 3.516595473115344, + "tokens_seen": 1497497600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027592778335005017, + "loss": 2.6545, + "theoretical_loss": 3.516595473115344, + "tokens_seen": 1497497600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027591775325977935, + "loss": 2.5794, + "theoretical_loss": 3.5165818999278535, + "tokens_seen": 1497563136 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027590772316950853, + "loss": 2.6723, + "theoretical_loss": 3.5165683275006447, + "tokens_seen": 1497628672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758976930792377, + "loss": 2.8326, + "theoretical_loss": 3.516554755833641, + "tokens_seen": 1497694208 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758876629889669, + "loss": 2.6049, + "theoretical_loss": 3.5165411849267683, + "tokens_seen": 1497759744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758776328986961, + "loss": 2.7356, + "theoretical_loss": 3.5165276147799487, + "tokens_seen": 1497825280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027586760280842525, + "loss": 2.7711, + "theoretical_loss": 3.516514045393108, + "tokens_seen": 1497890816 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758575727181545, + "loss": 2.611, + "theoretical_loss": 3.516500476766169, + "tokens_seen": 1497956352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758475426278836, + "loss": 2.653, + "theoretical_loss": 3.516486908899057, + "tokens_seen": 1498021888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027583751253761285, + "loss": 2.484, + "theoretical_loss": 3.516473341791696, + "tokens_seen": 1498087424 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027582748244734203, + "loss": 2.5086, + "theoretical_loss": 3.5164597754440097, + "tokens_seen": 1498152960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758174523570712, + "loss": 2.4447, + "theoretical_loss": 3.516446209855922, + "tokens_seen": 1498218496 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758074222668004, + "loss": 2.4562, + "theoretical_loss": 3.5164326450273586, + "tokens_seen": 1498284032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027579739217652963, + "loss": 2.6962, + "theoretical_loss": 3.5164190809582427, + "tokens_seen": 1498349568 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027578736208625876, + "loss": 2.6403, + "theoretical_loss": 3.516405517648499, + "tokens_seen": 1498415104 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275777331995988, + "loss": 2.6789, + "theoretical_loss": 3.516391955098052, + "tokens_seen": 1498480640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757673019057171, + "loss": 2.5963, + "theoretical_loss": 3.516378393306825, + "tokens_seen": 1498546176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027575727181544635, + "loss": 2.6649, + "theoretical_loss": 3.5163648322747427, + "tokens_seen": 1498611712 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027574724172517554, + "loss": 2.6449, + "theoretical_loss": 3.51635127200173, + "tokens_seen": 1498677248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757372116349047, + "loss": 2.8132, + "theoretical_loss": 3.5163377124877107, + "tokens_seen": 1498742784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757271815446339, + "loss": 2.698, + "theoretical_loss": 3.516324153732609, + "tokens_seen": 1498808320 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757171514543631, + "loss": 2.5638, + "theoretical_loss": 3.51631059573635, + "tokens_seen": 1498873856 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027570712136409226, + "loss": 2.6337, + "theoretical_loss": 3.516297038498857, + "tokens_seen": 1498939392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756970912738215, + "loss": 2.6763, + "theoretical_loss": 3.5162834820200555, + "tokens_seen": 1499004928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756870611835506, + "loss": 2.6062, + "theoretical_loss": 3.516269926299869, + "tokens_seen": 1499070464 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 827773, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0328586101531982, + "objective/train/theoretical_loss": 3.5162563713382218, + "objective/train/tokens_used": 1519596000, + "theoretical_loss": 3.5162563713382218, + "tokens_seen": 1499136000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027567703109327986, + "loss": 2.8939, + "theoretical_loss": 3.5162563713382218, + "tokens_seen": 1499136000 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275667001003009, + "loss": 2.5476, + "theoretical_loss": 3.516242817135039, + "tokens_seen": 1499201536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756569709127382, + "loss": 2.8686, + "theoretical_loss": 3.516229263690245, + "tokens_seen": 1499267072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756469408224674, + "loss": 2.7172, + "theoretical_loss": 3.5162157110037633, + "tokens_seen": 1499332608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756369107321966, + "loss": 2.6608, + "theoretical_loss": 3.516202159075519, + "tokens_seen": 1499398144 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027562688064192576, + "loss": 2.5377, + "theoretical_loss": 3.5161886079054367, + "tokens_seen": 1499463680 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275616850551655, + "loss": 2.884, + "theoretical_loss": 3.5161750574934407, + "tokens_seen": 1499529216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756068204613841, + "loss": 2.8149, + "theoretical_loss": 3.5161615078394552, + "tokens_seen": 1499594752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027559679037111336, + "loss": 2.6367, + "theoretical_loss": 3.516147958943405, + "tokens_seen": 1499660288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755867602808425, + "loss": 2.7369, + "theoretical_loss": 3.516134410805214, + "tokens_seen": 1499725824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755767301905717, + "loss": 2.5466, + "theoretical_loss": 3.5161208634248076, + "tokens_seen": 1499791360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755667001003009, + "loss": 2.6547, + "theoretical_loss": 3.5161073168021098, + "tokens_seen": 1499856896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755566700100301, + "loss": 2.5219, + "theoretical_loss": 3.5160937709370446, + "tokens_seen": 1499922432 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027554663991975927, + "loss": 2.7234, + "theoretical_loss": 3.516080225829538, + "tokens_seen": 1499987968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027553660982948845, + "loss": 2.635, + "theoretical_loss": 3.5160666814795127, + "tokens_seen": 1500053504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027552657973921763, + "loss": 2.6421, + "theoretical_loss": 3.5160531378868947, + "tokens_seen": 1500119040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027551654964894686, + "loss": 2.6204, + "theoretical_loss": 3.516039595051608, + "tokens_seen": 1500184576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027550651955867604, + "loss": 2.4071, + "theoretical_loss": 3.5160260529735767, + "tokens_seen": 1500250112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754964894684052, + "loss": 2.6619, + "theoretical_loss": 3.5160125116527263, + "tokens_seen": 1500315648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027548645937813446, + "loss": 2.7885, + "theoretical_loss": 3.515998971088981, + "tokens_seen": 1500381184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754764292878636, + "loss": 2.6149, + "theoretical_loss": 3.515985431282265, + "tokens_seen": 1500446720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754663991975928, + "loss": 2.7472, + "theoretical_loss": 3.515971892232504, + "tokens_seen": 1500512256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027545636910732195, + "loss": 2.4303, + "theoretical_loss": 3.515958353939621, + "tokens_seen": 1500577792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754463390170512, + "loss": 2.7099, + "theoretical_loss": 3.5159448164035423, + "tokens_seen": 1500643328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027543630892678037, + "loss": 2.8133, + "theoretical_loss": 3.5159312796241915, + "tokens_seen": 1500708864 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 828910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.892310380935669, + "objective/train/theoretical_loss": 3.5159177436014932, + "objective/train/tokens_used": 1521234400, + "theoretical_loss": 3.5159177436014932, + "tokens_seen": 1500774400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027542627883650955, + "loss": 2.6749, + "theoretical_loss": 3.5159177436014932, + "tokens_seen": 1500774400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027541624874623873, + "loss": 2.5888, + "theoretical_loss": 3.515904208335373, + "tokens_seen": 1500839936 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754062186559679, + "loss": 2.7679, + "theoretical_loss": 3.515890673825754, + "tokens_seen": 1500905472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753961885656971, + "loss": 2.7315, + "theoretical_loss": 3.515877140072563, + "tokens_seen": 1500971008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753861584754263, + "loss": 2.6909, + "theoretical_loss": 3.515863607075723, + "tokens_seen": 1501036544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027537612838515545, + "loss": 2.575, + "theoretical_loss": 3.5158500748351593, + "tokens_seen": 1501102080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753660982948847, + "loss": 2.5857, + "theoretical_loss": 3.5158365433507965, + "tokens_seen": 1501167616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753560682046138, + "loss": 2.5885, + "theoretical_loss": 3.51582301262256, + "tokens_seen": 1501233152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027534603811434305, + "loss": 2.7469, + "theoretical_loss": 3.5158094826503734, + "tokens_seen": 1501298688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027533600802407223, + "loss": 2.6433, + "theoretical_loss": 3.5157959534341625, + "tokens_seen": 1501364224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753259779338014, + "loss": 2.7166, + "theoretical_loss": 3.515782424973851, + "tokens_seen": 1501429760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753159478435306, + "loss": 2.5621, + "theoretical_loss": 3.5157688972693646, + "tokens_seen": 1501495296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027530591775325983, + "loss": 2.6297, + "theoretical_loss": 3.515755370320628, + "tokens_seen": 1501560832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027529588766298896, + "loss": 2.5006, + "theoretical_loss": 3.5157418441275654, + "tokens_seen": 1501626368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752858575727182, + "loss": 2.7195, + "theoretical_loss": 3.515728318690102, + "tokens_seen": 1501691904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752758274824473, + "loss": 2.7733, + "theoretical_loss": 3.5157147940081623, + "tokens_seen": 1501757440 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027526579739217655, + "loss": 2.8983, + "theoretical_loss": 3.515701270081672, + "tokens_seen": 1501822976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027525576730190574, + "loss": 2.5811, + "theoretical_loss": 3.515687746910555, + "tokens_seen": 1501888512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752457372116349, + "loss": 2.7298, + "theoretical_loss": 3.5156742244947363, + "tokens_seen": 1501954048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752357071213641, + "loss": 2.8053, + "theoretical_loss": 3.5156607028341416, + "tokens_seen": 1502019584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752256770310933, + "loss": 2.6385, + "theoretical_loss": 3.5156471819286947, + "tokens_seen": 1502085120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027521564694082246, + "loss": 2.5712, + "theoretical_loss": 3.5156336617783213, + "tokens_seen": 1502150656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752056168505517, + "loss": 2.7582, + "theoretical_loss": 3.515620142382945, + "tokens_seen": 1502216192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751955867602808, + "loss": 2.658, + "theoretical_loss": 3.5156066237424923, + "tokens_seen": 1502281728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027518555667001006, + "loss": 2.5665, + "theoretical_loss": 3.5155931058568877, + "tokens_seen": 1502347264 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 830192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0300025939941406, + "objective/train/theoretical_loss": 3.5155795887260552, + "objective/train/tokens_used": 1522872800, + "theoretical_loss": 3.5155795887260552, + "tokens_seen": 1502412800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751755265797392, + "loss": 2.5777, + "theoretical_loss": 3.5155795887260552, + "tokens_seen": 1502412800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751654964894684, + "loss": 2.6391, + "theoretical_loss": 3.5155660723499205, + "tokens_seen": 1502478336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751554663991976, + "loss": 2.6167, + "theoretical_loss": 3.5155525567284087, + "tokens_seen": 1502543872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751454363089268, + "loss": 2.8656, + "theoretical_loss": 3.5155390418614445, + "tokens_seen": 1502609408 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027513540621865596, + "loss": 2.7072, + "theoretical_loss": 3.5155255277489528, + "tokens_seen": 1502674944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751253761283852, + "loss": 2.7833, + "theoretical_loss": 3.5155120143908585, + "tokens_seen": 1502740480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751153460381143, + "loss": 2.5559, + "theoretical_loss": 3.5154985017870866, + "tokens_seen": 1502806016 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027510531594784356, + "loss": 2.6557, + "theoretical_loss": 3.515484989937563, + "tokens_seen": 1502871552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750952858575727, + "loss": 2.6358, + "theoretical_loss": 3.515471478842211, + "tokens_seen": 1502937088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750852557673019, + "loss": 2.5667, + "theoretical_loss": 3.5154579685009573, + "tokens_seen": 1503002624 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750752256770311, + "loss": 2.6918, + "theoretical_loss": 3.5154444589137253, + "tokens_seen": 1503068160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750651955867603, + "loss": 2.4683, + "theoretical_loss": 3.5154309500804417, + "tokens_seen": 1503133696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027505516549648947, + "loss": 2.6315, + "theoretical_loss": 3.5154174420010307, + "tokens_seen": 1503199232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027504513540621865, + "loss": 2.7197, + "theoretical_loss": 3.515403934675417, + "tokens_seen": 1503264768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027503510531594783, + "loss": 2.8134, + "theoretical_loss": 3.515390428103527, + "tokens_seen": 1503330304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027502507522567706, + "loss": 2.6432, + "theoretical_loss": 3.5153769222852844, + "tokens_seen": 1503395840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750150451354062, + "loss": 2.8172, + "theoretical_loss": 3.5153634172206143, + "tokens_seen": 1503461376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750050150451354, + "loss": 2.6194, + "theoretical_loss": 3.5153499129094428, + "tokens_seen": 1503526912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027499498495486455, + "loss": 2.6861, + "theoretical_loss": 3.515336409351695, + "tokens_seen": 1503592448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749849548645938, + "loss": 2.7088, + "theoretical_loss": 3.5153229065472953, + "tokens_seen": 1503657984 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027497492477432297, + "loss": 2.5598, + "theoretical_loss": 3.515309404496169, + "tokens_seen": 1503723520 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027496489468405215, + "loss": 2.8307, + "theoretical_loss": 3.515295903198241, + "tokens_seen": 1503789056 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027495486459378133, + "loss": 2.7083, + "theoretical_loss": 3.5152824026534373, + "tokens_seen": 1503854592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027494483450351057, + "loss": 2.5902, + "theoretical_loss": 3.515268902861682, + "tokens_seen": 1503920128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749348044132397, + "loss": 2.4284, + "theoretical_loss": 3.515255403822902, + "tokens_seen": 1503985664 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 830854, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.144463539123535, + "objective/train/theoretical_loss": 3.5152419055370205, + "objective/train/tokens_used": 1524511200, + "theoretical_loss": 3.5152419055370205, + "tokens_seen": 1504051200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027492477432296893, + "loss": 2.6293, + "theoretical_loss": 3.5152419055370205, + "tokens_seen": 1504051200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027491474423269806, + "loss": 2.597, + "theoretical_loss": 3.515228408003964, + "tokens_seen": 1504116736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749047141424273, + "loss": 2.5466, + "theoretical_loss": 3.515214911223657, + "tokens_seen": 1504182272 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027489468405215647, + "loss": 2.5915, + "theoretical_loss": 3.5152014151960254, + "tokens_seen": 1504247808 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027488465396188565, + "loss": 2.7626, + "theoretical_loss": 3.5151879199209937, + "tokens_seen": 1504313344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027487462387161483, + "loss": 2.6936, + "theoretical_loss": 3.515174425398487, + "tokens_seen": 1504378880 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274864593781344, + "loss": 2.6766, + "theoretical_loss": 3.515160931628432, + "tokens_seen": 1504444416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002748545636910732, + "loss": 2.6261, + "theoretical_loss": 3.515147438610753, + "tokens_seen": 1504509952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027484453360080243, + "loss": 2.4522, + "theoretical_loss": 3.515133946345375, + "tokens_seen": 1504575488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027483450351053156, + "loss": 2.7393, + "theoretical_loss": 3.515120454832223, + "tokens_seen": 1504641024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002748244734202608, + "loss": 2.5249, + "theoretical_loss": 3.515106964071224, + "tokens_seen": 1504706560 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027481444332999, + "loss": 2.7539, + "theoretical_loss": 3.5150934740623017, + "tokens_seen": 1504772096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027480441323971916, + "loss": 2.6781, + "theoretical_loss": 3.5150799848053818, + "tokens_seen": 1504837632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027479438314944834, + "loss": 2.7024, + "theoretical_loss": 3.51506649630039, + "tokens_seen": 1504903168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747843530591775, + "loss": 2.6632, + "theoretical_loss": 3.515053008547251, + "tokens_seen": 1504968704 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027477432296890675, + "loss": 2.668, + "theoretical_loss": 3.5150395215458907, + "tokens_seen": 1505034240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027476429287863594, + "loss": 2.5524, + "theoretical_loss": 3.515026035296235, + "tokens_seen": 1505099776 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747542627883651, + "loss": 2.5007, + "theoretical_loss": 3.515012549798208, + "tokens_seen": 1505165312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747442326980943, + "loss": 2.5512, + "theoretical_loss": 3.514999065051736, + "tokens_seen": 1505230848 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747342026078235, + "loss": 2.7295, + "theoretical_loss": 3.5149855810567434, + "tokens_seen": 1505296384 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027472417251755266, + "loss": 2.7503, + "theoretical_loss": 3.514972097813157, + "tokens_seen": 1505361920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747141424272819, + "loss": 2.6382, + "theoretical_loss": 3.514958615320901, + "tokens_seen": 1505427456 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274704112337011, + "loss": 2.7362, + "theoretical_loss": 3.514945133579902, + "tokens_seen": 1505492992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027469408224674026, + "loss": 2.8275, + "theoretical_loss": 3.514931652590084, + "tokens_seen": 1505558528 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746840521564694, + "loss": 2.6193, + "theoretical_loss": 3.514918172351374, + "tokens_seen": 1505624064 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 831840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.831157684326172, + "objective/train/theoretical_loss": 3.514904692863696, + "objective/train/tokens_used": 1526149600, + "theoretical_loss": 3.514904692863696, + "tokens_seen": 1505689600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746740220661986, + "loss": 2.6352, + "theoretical_loss": 3.514904692863696, + "tokens_seen": 1505689600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746639919759278, + "loss": 2.729, + "theoretical_loss": 3.514891214126976, + "tokens_seen": 1505755136 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274653961885657, + "loss": 2.7844, + "theoretical_loss": 3.5148777361411403, + "tokens_seen": 1505820672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027464393179538616, + "loss": 2.6508, + "theoretical_loss": 3.5148642589061136, + "tokens_seen": 1505886208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746339017051154, + "loss": 2.7491, + "theoretical_loss": 3.5148507824218216, + "tokens_seen": 1505951744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746238716148445, + "loss": 2.8469, + "theoretical_loss": 3.5148373066881895, + "tokens_seen": 1506017280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027461384152457376, + "loss": 2.4959, + "theoretical_loss": 3.5148238317051432, + "tokens_seen": 1506082816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746038114343029, + "loss": 2.4439, + "theoretical_loss": 3.5148103574726077, + "tokens_seen": 1506148352 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745937813440321, + "loss": 2.4271, + "theoretical_loss": 3.5147968839905097, + "tokens_seen": 1506213888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745837512537613, + "loss": 2.6857, + "theoretical_loss": 3.5147834112587737, + "tokens_seen": 1506279424 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745737211634905, + "loss": 2.6605, + "theoretical_loss": 3.5147699392773255, + "tokens_seen": 1506344960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027456369107321967, + "loss": 2.5731, + "theoretical_loss": 3.5147564680460905, + "tokens_seen": 1506410496 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027455366098294885, + "loss": 2.4873, + "theoretical_loss": 3.514742997564995, + "tokens_seen": 1506476032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027454363089267803, + "loss": 2.5501, + "theoretical_loss": 3.5147295278339636, + "tokens_seen": 1506541568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027453360080240726, + "loss": 2.7184, + "theoretical_loss": 3.514716058852923, + "tokens_seen": 1506607104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745235707121364, + "loss": 2.649, + "theoretical_loss": 3.514702590621798, + "tokens_seen": 1506672640 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745135406218656, + "loss": 2.7671, + "theoretical_loss": 3.5146891231405144, + "tokens_seen": 1506738176 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027450351053159475, + "loss": 2.4505, + "theoretical_loss": 3.514675656408998, + "tokens_seen": 1506803712 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274493480441324, + "loss": 2.7606, + "theoretical_loss": 3.5146621904271744, + "tokens_seen": 1506869248 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027448345035105317, + "loss": 2.5961, + "theoretical_loss": 3.514648725194969, + "tokens_seen": 1506934784 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027447342026078235, + "loss": 2.4571, + "theoretical_loss": 3.5146352607123084, + "tokens_seen": 1507000320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027446339017051153, + "loss": 2.6324, + "theoretical_loss": 3.5146217969791174, + "tokens_seen": 1507065856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027445336008024077, + "loss": 2.7367, + "theoretical_loss": 3.514608333995321, + "tokens_seen": 1507131392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744433299899699, + "loss": 2.7612, + "theoretical_loss": 3.514594871760847, + "tokens_seen": 1507196928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027443329989969913, + "loss": 2.6376, + "theoretical_loss": 3.5145814102756194, + "tokens_seen": 1507262464 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 832546, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4045536518096924, + "objective/train/theoretical_loss": 3.5145679495395643, + "objective/train/tokens_used": 1527788000, + "theoretical_loss": 3.5145679495395643, + "tokens_seen": 1507328000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027442326980942826, + "loss": 2.5427, + "theoretical_loss": 3.5145679495395643, + "tokens_seen": 1507328000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744132397191575, + "loss": 2.8226, + "theoretical_loss": 3.5145544895526077, + "tokens_seen": 1507393536 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027440320962888667, + "loss": 2.6932, + "theoretical_loss": 3.5145410303146756, + "tokens_seen": 1507459072 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027439317953861585, + "loss": 2.7651, + "theoretical_loss": 3.5145275718256928, + "tokens_seen": 1507524608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027438314944834503, + "loss": 2.4618, + "theoretical_loss": 3.514514114085586, + "tokens_seen": 1507590144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743731193580742, + "loss": 2.6122, + "theoretical_loss": 3.5145006570942803, + "tokens_seen": 1507655680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743630892678034, + "loss": 2.6113, + "theoretical_loss": 3.5144872008517023, + "tokens_seen": 1507721216 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027435305917753263, + "loss": 2.7369, + "theoretical_loss": 3.514473745357777, + "tokens_seen": 1507786752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027434302908726176, + "loss": 2.6171, + "theoretical_loss": 3.5144602906124307, + "tokens_seen": 1507852288 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274332998996991, + "loss": 2.6005, + "theoretical_loss": 3.5144468366155888, + "tokens_seen": 1507917824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743229689067202, + "loss": 2.724, + "theoretical_loss": 3.5144333833671775, + "tokens_seen": 1507983360 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027431293881644936, + "loss": 2.8599, + "theoretical_loss": 3.5144199308671227, + "tokens_seen": 1508048896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027430290872617854, + "loss": 2.6357, + "theoretical_loss": 3.5144064791153498, + "tokens_seen": 1508114432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742928786359077, + "loss": 2.642, + "theoretical_loss": 3.514393028111785, + "tokens_seen": 1508179968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742828485456369, + "loss": 2.7981, + "theoretical_loss": 3.5143795778563542, + "tokens_seen": 1508245504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027427281845536614, + "loss": 2.6059, + "theoretical_loss": 3.5143661283489833, + "tokens_seen": 1508311040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027426278836509526, + "loss": 2.5728, + "theoretical_loss": 3.514352679589598, + "tokens_seen": 1508376576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742527582748245, + "loss": 2.608, + "theoretical_loss": 3.5143392315781243, + "tokens_seen": 1508442112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742427281845536, + "loss": 2.5706, + "theoretical_loss": 3.5143257843144884, + "tokens_seen": 1508507648 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027423269809428286, + "loss": 2.5239, + "theoretical_loss": 3.514312337798615, + "tokens_seen": 1508573184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027422266800401204, + "loss": 2.4601, + "theoretical_loss": 3.514298892030432, + "tokens_seen": 1508638720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742126379137412, + "loss": 2.5261, + "theoretical_loss": 3.514285447009864, + "tokens_seen": 1508704256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742026078234704, + "loss": 2.6423, + "theoretical_loss": 3.514272002736837, + "tokens_seen": 1508769792 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741925777331996, + "loss": 2.6611, + "theoretical_loss": 3.5142585592112776, + "tokens_seen": 1508835328 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027418254764292877, + "loss": 2.825, + "theoretical_loss": 3.514245116433111, + "tokens_seen": 1508900864 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 833973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9619827270507812, + "objective/train/theoretical_loss": 3.5142316744022644, + "objective/train/tokens_used": 1529426400, + "theoretical_loss": 3.5142316744022644, + "tokens_seen": 1508966400 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274172517552658, + "loss": 2.6156, + "theoretical_loss": 3.5142316744022644, + "tokens_seen": 1508966400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027416248746238713, + "loss": 2.5253, + "theoretical_loss": 3.5142182331186627, + "tokens_seen": 1509031936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027415245737211636, + "loss": 2.5278, + "theoretical_loss": 3.5142047925822317, + "tokens_seen": 1509097472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027414242728184554, + "loss": 2.8177, + "theoretical_loss": 3.5141913527928983, + "tokens_seen": 1509163008 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741323971915747, + "loss": 2.5251, + "theoretical_loss": 3.514177913750588, + "tokens_seen": 1509228544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741223671013039, + "loss": 2.639, + "theoretical_loss": 3.5141644754552277, + "tokens_seen": 1509294080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741123370110331, + "loss": 2.4835, + "theoretical_loss": 3.514151037906742, + "tokens_seen": 1509359616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027410230692076227, + "loss": 2.6622, + "theoretical_loss": 3.514137601105058, + "tokens_seen": 1509425152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740922768304915, + "loss": 2.5462, + "theoretical_loss": 3.514124165050102, + "tokens_seen": 1509490688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027408224674022063, + "loss": 2.6131, + "theoretical_loss": 3.514110729741799, + "tokens_seen": 1509556224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027407221664994987, + "loss": 2.6414, + "theoretical_loss": 3.5140972951800764, + "tokens_seen": 1509621760 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274062186559679, + "loss": 2.6016, + "theoretical_loss": 3.514083861364859, + "tokens_seen": 1509687296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027405215646940823, + "loss": 2.5222, + "theoretical_loss": 3.5140704282960735, + "tokens_seen": 1509752832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740421263791374, + "loss": 2.4032, + "theoretical_loss": 3.514056995973646, + "tokens_seen": 1509818368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740320962888666, + "loss": 2.6077, + "theoretical_loss": 3.514043564397503, + "tokens_seen": 1509883904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740220661985958, + "loss": 2.7701, + "theoretical_loss": 3.5140301335675703, + "tokens_seen": 1509949440 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027401203610832495, + "loss": 2.7075, + "theoretical_loss": 3.514016703483774, + "tokens_seen": 1510014976 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740020060180542, + "loss": 2.6965, + "theoretical_loss": 3.5140032741460403, + "tokens_seen": 1510080512 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027399197592778337, + "loss": 2.7415, + "theoretical_loss": 3.5139898455542955, + "tokens_seen": 1510146048 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027398194583751255, + "loss": 2.4527, + "theoretical_loss": 3.5139764177084656, + "tokens_seen": 1510211584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027397191574724173, + "loss": 2.6699, + "theoretical_loss": 3.5139629906084773, + "tokens_seen": 1510277120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027396188565697097, + "loss": 2.5838, + "theoretical_loss": 3.513949564254256, + "tokens_seen": 1510342656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002739518555667001, + "loss": 2.7862, + "theoretical_loss": 3.513936138645729, + "tokens_seen": 1510408192 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027394182547642933, + "loss": 2.8045, + "theoretical_loss": 3.5139227137828213, + "tokens_seen": 1510473728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027393179538615846, + "loss": 2.8602, + "theoretical_loss": 3.5139092896654596, + "tokens_seen": 1510539264 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 834645, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1922924518585205, + "objective/train/theoretical_loss": 3.513895866293571, + "objective/train/tokens_used": 1531064800, + "theoretical_loss": 3.513895866293571, + "tokens_seen": 1510604800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002739217652958877, + "loss": 2.5753, + "theoretical_loss": 3.513895866293571, + "tokens_seen": 1510604800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027391173520561687, + "loss": 2.3657, + "theoretical_loss": 3.5138824436670806, + "tokens_seen": 1510670336 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027390170511534605, + "loss": 2.5346, + "theoretical_loss": 3.513869021785915, + "tokens_seen": 1510735872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027389167502507524, + "loss": 2.6478, + "theoretical_loss": 3.513855600650001, + "tokens_seen": 1510801408 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738816449348044, + "loss": 2.4694, + "theoretical_loss": 3.513842180259264, + "tokens_seen": 1510866944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738716148445336, + "loss": 3.0212, + "theoretical_loss": 3.513828760613631, + "tokens_seen": 1510932480 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027386158475426283, + "loss": 2.6099, + "theoretical_loss": 3.5138153417130287, + "tokens_seen": 1510998016 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027385155466399196, + "loss": 2.6894, + "theoretical_loss": 3.513801923557382, + "tokens_seen": 1511063552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738415245737212, + "loss": 2.5968, + "theoretical_loss": 3.5137885061466188, + "tokens_seen": 1511129088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738314944834504, + "loss": 2.6717, + "theoretical_loss": 3.5137750894806645, + "tokens_seen": 1511194624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027382146439317956, + "loss": 2.6688, + "theoretical_loss": 3.5137616735594452, + "tokens_seen": 1511260160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027381143430290874, + "loss": 2.6462, + "theoretical_loss": 3.513748258382888, + "tokens_seen": 1511325696 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738014042126379, + "loss": 2.5841, + "theoretical_loss": 3.5137348439509193, + "tokens_seen": 1511391232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737913741223671, + "loss": 2.433, + "theoretical_loss": 3.513721430263465, + "tokens_seen": 1511456768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027378134403209634, + "loss": 2.7375, + "theoretical_loss": 3.513708017320452, + "tokens_seen": 1511522304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027377131394182546, + "loss": 2.465, + "theoretical_loss": 3.5136946051218065, + "tokens_seen": 1511587840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737612838515547, + "loss": 2.4808, + "theoretical_loss": 3.5136811936674546, + "tokens_seen": 1511653376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737512537612838, + "loss": 2.5119, + "theoretical_loss": 3.5136677829573233, + "tokens_seen": 1511718912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027374122367101306, + "loss": 2.5002, + "theoretical_loss": 3.5136543729913385, + "tokens_seen": 1511784448 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027373119358074224, + "loss": 2.5367, + "theoretical_loss": 3.5136409637694266, + "tokens_seen": 1511849984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737211634904714, + "loss": 2.5183, + "theoretical_loss": 3.5136275552915146, + "tokens_seen": 1511915520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737111334002006, + "loss": 2.6877, + "theoretical_loss": 3.5136141475575293, + "tokens_seen": 1511981056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737011033099298, + "loss": 2.8269, + "theoretical_loss": 3.513600740567396, + "tokens_seen": 1512046592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027369107321965897, + "loss": 2.6315, + "theoretical_loss": 3.5135873343210418, + "tokens_seen": 1512112128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736810431293882, + "loss": 2.6766, + "theoretical_loss": 3.5135739288183934, + "tokens_seen": 1512177664 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 835710, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6051583290100098, + "objective/train/theoretical_loss": 3.513560524059377, + "objective/train/tokens_used": 1532703200, + "theoretical_loss": 3.513560524059377, + "tokens_seen": 1512243200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027367101303911733, + "loss": 2.7525, + "theoretical_loss": 3.513560524059377, + "tokens_seen": 1512243200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027366098294884656, + "loss": 2.6414, + "theoretical_loss": 3.513547120043919, + "tokens_seen": 1512308736 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027365095285857574, + "loss": 2.5715, + "theoretical_loss": 3.5135337167719465, + "tokens_seen": 1512374272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736409227683049, + "loss": 2.872, + "theoretical_loss": 3.5135203142433857, + "tokens_seen": 1512439808 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736308926780341, + "loss": 2.5917, + "theoretical_loss": 3.513506912458163, + "tokens_seen": 1512505344 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736208625877633, + "loss": 2.5591, + "theoretical_loss": 3.5134935114162054, + "tokens_seen": 1512570880 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027361083249749247, + "loss": 2.6358, + "theoretical_loss": 3.513480111117439, + "tokens_seen": 1512636416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736008024072217, + "loss": 2.3513, + "theoretical_loss": 3.5134667115617906, + "tokens_seen": 1512701952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027359077231695083, + "loss": 2.6456, + "theoretical_loss": 3.513453312749187, + "tokens_seen": 1512767488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027358074222668007, + "loss": 2.3847, + "theoretical_loss": 3.5134399146795543, + "tokens_seen": 1512833024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735707121364092, + "loss": 2.5213, + "theoretical_loss": 3.51342651735282, + "tokens_seen": 1512898560 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027356068204613843, + "loss": 2.5991, + "theoretical_loss": 3.51341312076891, + "tokens_seen": 1512964096 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735506519558676, + "loss": 2.7351, + "theoretical_loss": 3.5133997249277504, + "tokens_seen": 1513029632 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735406218655968, + "loss": 2.5695, + "theoretical_loss": 3.5133863298292694, + "tokens_seen": 1513095168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027353059177532597, + "loss": 2.6775, + "theoretical_loss": 3.5133729354733925, + "tokens_seen": 1513160704 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027352056168505515, + "loss": 2.5956, + "theoretical_loss": 3.513359541860047, + "tokens_seen": 1513226240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027351053159478433, + "loss": 2.5487, + "theoretical_loss": 3.5133461489891586, + "tokens_seen": 1513291776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027350050150451357, + "loss": 2.6104, + "theoretical_loss": 3.513332756860655, + "tokens_seen": 1513357312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734904714142427, + "loss": 2.371, + "theoretical_loss": 3.5133193654744623, + "tokens_seen": 1513422848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027348044132397193, + "loss": 2.7229, + "theoretical_loss": 3.513305974830508, + "tokens_seen": 1513488384 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734704112337011, + "loss": 2.7482, + "theoretical_loss": 3.513292584928718, + "tokens_seen": 1513553920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734603811434303, + "loss": 2.587, + "theoretical_loss": 3.5132791957690195, + "tokens_seen": 1513619456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734503510531595, + "loss": 2.487, + "theoretical_loss": 3.513265807351339, + "tokens_seen": 1513684992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027344032096288866, + "loss": 2.5462, + "theoretical_loss": 3.513252419675603, + "tokens_seen": 1513750528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027343029087261784, + "loss": 2.9182, + "theoretical_loss": 3.5132390327417395, + "tokens_seen": 1513816064 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 836217, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6275956630706787, + "objective/train/theoretical_loss": 3.5132256465496736, + "objective/train/tokens_used": 1534341600, + "theoretical_loss": 3.5132256465496736, + "tokens_seen": 1513881600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734202607823471, + "loss": 2.5214, + "theoretical_loss": 3.5132256465496736, + "tokens_seen": 1513881600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734102306920762, + "loss": 2.5889, + "theoretical_loss": 3.513212261099333, + "tokens_seen": 1513947136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027340020060180544, + "loss": 2.7688, + "theoretical_loss": 3.5131988763906445, + "tokens_seen": 1514012672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027339017051153456, + "loss": 2.6498, + "theoretical_loss": 3.513185492423535, + "tokens_seen": 1514078208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733801404212638, + "loss": 2.4727, + "theoretical_loss": 3.5131721091979307, + "tokens_seen": 1514143744 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273370110330993, + "loss": 2.6173, + "theoretical_loss": 3.5131587267137587, + "tokens_seen": 1514209280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027336008024072216, + "loss": 2.5951, + "theoretical_loss": 3.5131453449709458, + "tokens_seen": 1514274816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027335005015045134, + "loss": 2.379, + "theoretical_loss": 3.5131319639694194, + "tokens_seen": 1514340352 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733400200601806, + "loss": 2.6354, + "theoretical_loss": 3.513118583709106, + "tokens_seen": 1514405888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733299899699097, + "loss": 2.7082, + "theoretical_loss": 3.513105204189932, + "tokens_seen": 1514471424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027331995987963894, + "loss": 2.7024, + "theoretical_loss": 3.5130918254118253, + "tokens_seen": 1514536960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027330992978936807, + "loss": 2.4955, + "theoretical_loss": 3.5130784473747116, + "tokens_seen": 1514602496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732998996990973, + "loss": 2.5076, + "theoretical_loss": 3.5130650700785186, + "tokens_seen": 1514668032 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732898696088265, + "loss": 2.4884, + "theoretical_loss": 3.513051693523173, + "tokens_seen": 1514733568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027327983951855566, + "loss": 2.4027, + "theoretical_loss": 3.5130383177086015, + "tokens_seen": 1514799104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732698094282849, + "loss": 2.579, + "theoretical_loss": 3.5130249426347318, + "tokens_seen": 1514864640 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273259779338014, + "loss": 2.6875, + "theoretical_loss": 3.51301156830149, + "tokens_seen": 1514930176 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027324974924774326, + "loss": 2.6929, + "theoretical_loss": 3.512998194708803, + "tokens_seen": 1514995712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027323971915747244, + "loss": 2.5525, + "theoretical_loss": 3.512984821856598, + "tokens_seen": 1515061248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732296890672016, + "loss": 2.795, + "theoretical_loss": 3.5129714497448026, + "tokens_seen": 1515126784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732196589769308, + "loss": 2.4369, + "theoretical_loss": 3.5129580783733436, + "tokens_seen": 1515192320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027320962888666, + "loss": 2.6934, + "theoretical_loss": 3.5129447077421467, + "tokens_seen": 1515257856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027319959879638917, + "loss": 2.6587, + "theoretical_loss": 3.512931337851141, + "tokens_seen": 1515323392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731895687061184, + "loss": 2.7382, + "theoretical_loss": 3.5129179687002514, + "tokens_seen": 1515388928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027317953861584753, + "loss": 2.6136, + "theoretical_loss": 3.512904600289406, + "tokens_seen": 1515454464 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 837242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.814265251159668, + "objective/train/theoretical_loss": 3.512891232618532, + "objective/train/tokens_used": 1535980000, + "theoretical_loss": 3.512891232618532, + "tokens_seen": 1515520000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027316950852557676, + "loss": 2.709, + "theoretical_loss": 3.512891232618532, + "tokens_seen": 1515520000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027315947843530594, + "loss": 2.5236, + "theoretical_loss": 3.512877865687556, + "tokens_seen": 1515585536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731494483450351, + "loss": 2.8312, + "theoretical_loss": 3.5128644994964056, + "tokens_seen": 1515651072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731394182547643, + "loss": 2.6342, + "theoretical_loss": 3.512851134045007, + "tokens_seen": 1515716608 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731293881644935, + "loss": 2.3248, + "theoretical_loss": 3.512837769333288, + "tokens_seen": 1515782144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027311935807422267, + "loss": 2.5907, + "theoretical_loss": 3.512824405361175, + "tokens_seen": 1515847680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731093279839519, + "loss": 2.486, + "theoretical_loss": 3.5128110421285963, + "tokens_seen": 1515913216 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027309929789368103, + "loss": 2.561, + "theoretical_loss": 3.5127976796354776, + "tokens_seen": 1515978752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027308926780341027, + "loss": 2.8013, + "theoretical_loss": 3.5127843178817475, + "tokens_seen": 1516044288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002730792377131394, + "loss": 2.5745, + "theoretical_loss": 3.5127709568673318, + "tokens_seen": 1516109824 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027306920762286863, + "loss": 2.6471, + "theoretical_loss": 3.5127575965921576, + "tokens_seen": 1516175360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002730591775325978, + "loss": 2.5791, + "theoretical_loss": 3.5127442370561535, + "tokens_seen": 1516240896 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273049147442327, + "loss": 2.4793, + "theoretical_loss": 3.5127308782592452, + "tokens_seen": 1516306432 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027303911735205617, + "loss": 2.6887, + "theoretical_loss": 3.5127175202013605, + "tokens_seen": 1516371968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027302908726178535, + "loss": 2.8017, + "theoretical_loss": 3.512704162882427, + "tokens_seen": 1516437504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027301905717151453, + "loss": 2.6432, + "theoretical_loss": 3.5126908063023707, + "tokens_seen": 1516503040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027300902708124377, + "loss": 2.6128, + "theoretical_loss": 3.5126774504611196, + "tokens_seen": 1516568576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729989969909729, + "loss": 2.5469, + "theoretical_loss": 3.512664095358601, + "tokens_seen": 1516634112 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027298896690070213, + "loss": 2.6823, + "theoretical_loss": 3.512650740994742, + "tokens_seen": 1516699648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729789368104313, + "loss": 2.6485, + "theoretical_loss": 3.5126373873694696, + "tokens_seen": 1516765184 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729689067201605, + "loss": 2.6336, + "theoretical_loss": 3.512624034482711, + "tokens_seen": 1516830720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729588766298897, + "loss": 2.6261, + "theoretical_loss": 3.512610682334394, + "tokens_seen": 1516896256 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027294884653961886, + "loss": 2.7059, + "theoretical_loss": 3.512597330924445, + "tokens_seen": 1516961792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027293881644934804, + "loss": 2.5078, + "theoretical_loss": 3.512583980252792, + "tokens_seen": 1517027328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729287863590773, + "loss": 2.5923, + "theoretical_loss": 3.5125706303193622, + "tokens_seen": 1517092864 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 837775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.929088830947876, + "objective/train/theoretical_loss": 3.5125572811240833, + "objective/train/tokens_used": 1537618400, + "theoretical_loss": 3.5125572811240833, + "tokens_seen": 1517158400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729187562688064, + "loss": 2.7973, + "theoretical_loss": 3.5125572811240833, + "tokens_seen": 1517158400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027290872617853564, + "loss": 2.7104, + "theoretical_loss": 3.512543932666881, + "tokens_seen": 1517223936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027289869608826476, + "loss": 2.5193, + "theoretical_loss": 3.512530584947684, + "tokens_seen": 1517289472 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272888665997994, + "loss": 2.7381, + "theoretical_loss": 3.51251723796642, + "tokens_seen": 1517355008 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728786359077232, + "loss": 2.6233, + "theoretical_loss": 3.5125038917230147, + "tokens_seen": 1517420544 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027286860581745236, + "loss": 2.5533, + "theoretical_loss": 3.5124905462173963, + "tokens_seen": 1517486080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027285857572718154, + "loss": 2.5144, + "theoretical_loss": 3.512477201449493, + "tokens_seen": 1517551616 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728485456369108, + "loss": 2.6085, + "theoretical_loss": 3.512463857419231, + "tokens_seen": 1517617152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728385155466399, + "loss": 2.4266, + "theoretical_loss": 3.512450514126538, + "tokens_seen": 1517682688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027282848545636914, + "loss": 2.5963, + "theoretical_loss": 3.5124371715713414, + "tokens_seen": 1517748224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027281845536609827, + "loss": 2.6626, + "theoretical_loss": 3.5124238297535686, + "tokens_seen": 1517813760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728084252758275, + "loss": 2.674, + "theoretical_loss": 3.5124104886731473, + "tokens_seen": 1517879296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727983951855567, + "loss": 2.6903, + "theoretical_loss": 3.5123971483300043, + "tokens_seen": 1517944832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027278836509528586, + "loss": 2.3904, + "theoretical_loss": 3.512383808724068, + "tokens_seen": 1518010368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027277833500501504, + "loss": 2.5493, + "theoretical_loss": 3.5123704698552647, + "tokens_seen": 1518075904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727683049147442, + "loss": 2.5472, + "theoretical_loss": 3.5123571317235225, + "tokens_seen": 1518141440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727582748244734, + "loss": 2.7771, + "theoretical_loss": 3.5123437943287685, + "tokens_seen": 1518206976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027274824473420264, + "loss": 2.7707, + "theoretical_loss": 3.512330457670931, + "tokens_seen": 1518272512 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027273821464393177, + "loss": 2.6094, + "theoretical_loss": 3.5123171217499363, + "tokens_seen": 1518338048 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272728184553661, + "loss": 2.6912, + "theoretical_loss": 3.5123037865657127, + "tokens_seen": 1518403584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027271815446339013, + "loss": 2.4024, + "theoretical_loss": 3.5122904521181875, + "tokens_seen": 1518469120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027270812437311937, + "loss": 2.6141, + "theoretical_loss": 3.512277118407288, + "tokens_seen": 1518534656 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027269809428284855, + "loss": 2.7052, + "theoretical_loss": 3.512263785432942, + "tokens_seen": 1518600192 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027268806419257773, + "loss": 2.5501, + "theoretical_loss": 3.512250453195077, + "tokens_seen": 1518665728 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002726780341023069, + "loss": 2.6798, + "theoretical_loss": 3.5122371216936203, + "tokens_seen": 1518731264 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 838761, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3899428844451904, + "objective/train/theoretical_loss": 3.5122237909284997, + "objective/train/tokens_used": 1539256800, + "theoretical_loss": 3.5122237909284997, + "tokens_seen": 1518796800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027266800401203614, + "loss": 2.6283, + "theoretical_loss": 3.5122237909284997, + "tokens_seen": 1518796800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027265797392176527, + "loss": 2.6057, + "theoretical_loss": 3.5122104608996425, + "tokens_seen": 1518862336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002726479438314945, + "loss": 2.4512, + "theoretical_loss": 3.512197131606977, + "tokens_seen": 1518927872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027263791374122363, + "loss": 2.9519, + "theoretical_loss": 3.5121838030504295, + "tokens_seen": 1518993408 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027262788365095287, + "loss": 2.5698, + "theoretical_loss": 3.5121704752299285, + "tokens_seen": 1519058944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027261785356068205, + "loss": 2.7011, + "theoretical_loss": 3.512157148145402, + "tokens_seen": 1519124480 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027260782347041123, + "loss": 2.572, + "theoretical_loss": 3.5121438217967764, + "tokens_seen": 1519190016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725977933801404, + "loss": 2.7506, + "theoretical_loss": 3.51213049618398, + "tokens_seen": 1519255552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725877632898696, + "loss": 2.653, + "theoretical_loss": 3.5121171713069406, + "tokens_seen": 1519321088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725777331995988, + "loss": 2.878, + "theoretical_loss": 3.5121038471655854, + "tokens_seen": 1519386624 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272567703109328, + "loss": 2.5658, + "theoretical_loss": 3.5120905237598423, + "tokens_seen": 1519452160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027255767301905714, + "loss": 2.4503, + "theoretical_loss": 3.512077201089639, + "tokens_seen": 1519517696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027254764292878637, + "loss": 2.4628, + "theoretical_loss": 3.512063879154903, + "tokens_seen": 1519583232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725376128385155, + "loss": 2.6997, + "theoretical_loss": 3.512050557955562, + "tokens_seen": 1519648768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027252758274824473, + "loss": 2.7425, + "theoretical_loss": 3.512037237491544, + "tokens_seen": 1519714304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027251755265797397, + "loss": 2.4202, + "theoretical_loss": 3.512023917762776, + "tokens_seen": 1519779840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725075225677031, + "loss": 2.6174, + "theoretical_loss": 3.512010598769187, + "tokens_seen": 1519845376 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027249749247743233, + "loss": 2.4447, + "theoretical_loss": 3.5119972805107036, + "tokens_seen": 1519910912 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724874623871615, + "loss": 2.5532, + "theoretical_loss": 3.5119839629872533, + "tokens_seen": 1519976448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724774322968907, + "loss": 2.6757, + "theoretical_loss": 3.5119706461987654, + "tokens_seen": 1520041984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724674022066199, + "loss": 2.5968, + "theoretical_loss": 3.511957330145166, + "tokens_seen": 1520107520 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027245737211634906, + "loss": 2.6554, + "theoretical_loss": 3.5119440148263834, + "tokens_seen": 1520173056 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027244734202607824, + "loss": 2.7129, + "theoretical_loss": 3.5119307002423454, + "tokens_seen": 1520238592 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724373119358075, + "loss": 2.6401, + "theoretical_loss": 3.5119173863929807, + "tokens_seen": 1520304128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724272818455366, + "loss": 2.6772, + "theoretical_loss": 3.511904073278216, + "tokens_seen": 1520369664 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 839881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8261830806732178, + "objective/train/theoretical_loss": 3.5118907608979786, + "objective/train/tokens_used": 1540895200, + "theoretical_loss": 3.5118907608979786, + "tokens_seen": 1520435200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027241725175526584, + "loss": 2.734, + "theoretical_loss": 3.5118907608979786, + "tokens_seen": 1520435200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027240722166499496, + "loss": 2.7, + "theoretical_loss": 3.511877449252198, + "tokens_seen": 1520500736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723971915747242, + "loss": 2.5904, + "theoretical_loss": 3.5118641383408007, + "tokens_seen": 1520566272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723871614844534, + "loss": 2.6964, + "theoretical_loss": 3.511850828163715, + "tokens_seen": 1520631808 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027237713139418256, + "loss": 2.6544, + "theoretical_loss": 3.511837518720869, + "tokens_seen": 1520697344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027236710130391174, + "loss": 2.758, + "theoretical_loss": 3.5118242100121897, + "tokens_seen": 1520762880 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272357071213641, + "loss": 2.4927, + "theoretical_loss": 3.5118109020376056, + "tokens_seen": 1520828416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723470411233701, + "loss": 2.6839, + "theoretical_loss": 3.5117975947970446, + "tokens_seen": 1520893952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027233701103309934, + "loss": 2.6804, + "theoretical_loss": 3.5117842882904347, + "tokens_seen": 1520959488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027232698094282847, + "loss": 2.6058, + "theoretical_loss": 3.511770982517703, + "tokens_seen": 1521025024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723169508525577, + "loss": 2.4964, + "theoretical_loss": 3.5117576774787786, + "tokens_seen": 1521090560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723069207622869, + "loss": 2.4434, + "theoretical_loss": 3.5117443731735887, + "tokens_seen": 1521156096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027229689067201606, + "loss": 2.4178, + "theoretical_loss": 3.511731069602061, + "tokens_seen": 1521221632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027228686058174524, + "loss": 2.7692, + "theoretical_loss": 3.511717766764124, + "tokens_seen": 1521287168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722768304914744, + "loss": 2.7603, + "theoretical_loss": 3.5117044646597058, + "tokens_seen": 1521352704 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722668004012036, + "loss": 2.5612, + "theoretical_loss": 3.5116911632887335, + "tokens_seen": 1521418240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027225677031093284, + "loss": 2.4733, + "theoretical_loss": 3.511677862651136, + "tokens_seen": 1521483776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027224674022066197, + "loss": 2.7743, + "theoretical_loss": 3.5116645627468404, + "tokens_seen": 1521549312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722367101303912, + "loss": 2.889, + "theoretical_loss": 3.511651263575775, + "tokens_seen": 1521614848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027222668004012033, + "loss": 2.2772, + "theoretical_loss": 3.511637965137868, + "tokens_seen": 1521680384 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027221664994984957, + "loss": 2.4408, + "theoretical_loss": 3.511624667433048, + "tokens_seen": 1521745920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027220661985957875, + "loss": 2.5401, + "theoretical_loss": 3.5116113704612415, + "tokens_seen": 1521811456 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027219658976930793, + "loss": 2.636, + "theoretical_loss": 3.511598074222378, + "tokens_seen": 1521876992 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721865596790371, + "loss": 2.4016, + "theoretical_loss": 3.5115847787163843, + "tokens_seen": 1521942528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027217652958876635, + "loss": 2.5686, + "theoretical_loss": 3.5115714839431895, + "tokens_seen": 1522008064 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 840376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.44497013092041, + "objective/train/theoretical_loss": 3.511558189902721, + "objective/train/tokens_used": 1542533600, + "theoretical_loss": 3.511558189902721, + "tokens_seen": 1522073600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027216649949849547, + "loss": 2.5371, + "theoretical_loss": 3.511558189902721, + "tokens_seen": 1522073600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721564694082247, + "loss": 2.8807, + "theoretical_loss": 3.511544896594907, + "tokens_seen": 1522139136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027214643931795383, + "loss": 2.5307, + "theoretical_loss": 3.511531604019676, + "tokens_seen": 1522204672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027213640922768307, + "loss": 2.5698, + "theoretical_loss": 3.5115183121769555, + "tokens_seen": 1522270208 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027212637913741225, + "loss": 2.5847, + "theoretical_loss": 3.511505021066674, + "tokens_seen": 1522335744 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027211634904714143, + "loss": 2.515, + "theoretical_loss": 3.5114917306887596, + "tokens_seen": 1522401280 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721063189568706, + "loss": 2.7306, + "theoretical_loss": 3.51147844104314, + "tokens_seen": 1522466816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720962888665998, + "loss": 2.7946, + "theoretical_loss": 3.5114651521297437, + "tokens_seen": 1522532352 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272086258776329, + "loss": 2.625, + "theoretical_loss": 3.5114518639484986, + "tokens_seen": 1522597888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720762286860582, + "loss": 2.4889, + "theoretical_loss": 3.5114385764993337, + "tokens_seen": 1522663424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027206619859578734, + "loss": 2.8904, + "theoretical_loss": 3.511425289782176, + "tokens_seen": 1522728960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027205616850551657, + "loss": 2.716, + "theoretical_loss": 3.511412003796954, + "tokens_seen": 1522794496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720461384152457, + "loss": 2.6501, + "theoretical_loss": 3.511398718543596, + "tokens_seen": 1522860032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027203610832497493, + "loss": 2.6602, + "theoretical_loss": 3.5113854340220305, + "tokens_seen": 1522925568 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720260782347041, + "loss": 2.6252, + "theoretical_loss": 3.5113721502321855, + "tokens_seen": 1522991104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720160481444333, + "loss": 2.6795, + "theoretical_loss": 3.511358867173989, + "tokens_seen": 1523056640 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720060180541625, + "loss": 2.5826, + "theoretical_loss": 3.511345584847369, + "tokens_seen": 1523122176 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719959879638917, + "loss": 2.4883, + "theoretical_loss": 3.5113323032522548, + "tokens_seen": 1523187712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027198595787362084, + "loss": 2.5199, + "theoretical_loss": 3.511319022388574, + "tokens_seen": 1523253248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719759277833501, + "loss": 2.7365, + "theoretical_loss": 3.511305742256254, + "tokens_seen": 1523318784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719658976930792, + "loss": 2.5625, + "theoretical_loss": 3.5112924628552244, + "tokens_seen": 1523384320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027195586760280844, + "loss": 2.7394, + "theoretical_loss": 3.5112791841854127, + "tokens_seen": 1523449856 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719458375125376, + "loss": 2.6528, + "theoretical_loss": 3.5112659062467477, + "tokens_seen": 1523515392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719358074222668, + "loss": 2.6489, + "theoretical_loss": 3.511252629039157, + "tokens_seen": 1523580928 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271925777331996, + "loss": 2.7094, + "theoretical_loss": 3.5112393525625696, + "tokens_seen": 1523646464 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 841650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9947397708892822, + "objective/train/theoretical_loss": 3.5112260768169135, + "objective/train/tokens_used": 1544172000, + "theoretical_loss": 3.5112260768169135, + "tokens_seen": 1523712000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027191574724172516, + "loss": 2.7517, + "theoretical_loss": 3.5112260768169135, + "tokens_seen": 1523712000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027190571715145434, + "loss": 2.5517, + "theoretical_loss": 3.5112128018021167, + "tokens_seen": 1523777536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718956870611836, + "loss": 2.5746, + "theoretical_loss": 3.511199527518108, + "tokens_seen": 1523843072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718856569709127, + "loss": 2.4339, + "theoretical_loss": 3.5111862539648158, + "tokens_seen": 1523908608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027187562688064194, + "loss": 2.5428, + "theoretical_loss": 3.5111729811421686, + "tokens_seen": 1523974144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027186559679037107, + "loss": 2.8515, + "theoretical_loss": 3.511159709050094, + "tokens_seen": 1524039680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718555667001003, + "loss": 2.6896, + "theoretical_loss": 3.5111464376885206, + "tokens_seen": 1524105216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718455366098295, + "loss": 2.4945, + "theoretical_loss": 3.5111331670573773, + "tokens_seen": 1524170752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027183550651955867, + "loss": 2.8215, + "theoretical_loss": 3.5111198971565925, + "tokens_seen": 1524236288 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027182547642928785, + "loss": 2.657, + "theoretical_loss": 3.5111066279860936, + "tokens_seen": 1524301824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718154463390171, + "loss": 2.6308, + "theoretical_loss": 3.51109335954581, + "tokens_seen": 1524367360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718054162487462, + "loss": 2.6359, + "theoretical_loss": 3.5110800918356704, + "tokens_seen": 1524432896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027179538615847544, + "loss": 2.6419, + "theoretical_loss": 3.511066824855602, + "tokens_seen": 1524498432 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027178535606820457, + "loss": 2.5588, + "theoretical_loss": 3.511053558605534, + "tokens_seen": 1524563968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717753259779338, + "loss": 2.6852, + "theoretical_loss": 3.5110402930853946, + "tokens_seen": 1524629504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027176529588766304, + "loss": 2.6305, + "theoretical_loss": 3.511027028295113, + "tokens_seen": 1524695040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027175526579739217, + "loss": 2.7099, + "theoretical_loss": 3.5110137642346166, + "tokens_seen": 1524760576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717452357071214, + "loss": 2.5974, + "theoretical_loss": 3.5110005009038345, + "tokens_seen": 1524826112 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027173520561685053, + "loss": 2.5775, + "theoretical_loss": 3.510987238302695, + "tokens_seen": 1524891648 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027172517552657977, + "loss": 2.6237, + "theoretical_loss": 3.510973976431127, + "tokens_seen": 1524957184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027171514543630895, + "loss": 2.5999, + "theoretical_loss": 3.510960715289059, + "tokens_seen": 1525022720 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027170511534603813, + "loss": 2.7274, + "theoretical_loss": 3.5109474548764186, + "tokens_seen": 1525088256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716950852557673, + "loss": 2.625, + "theoretical_loss": 3.510934195193135, + "tokens_seen": 1525153792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027168505516549655, + "loss": 2.5496, + "theoretical_loss": 3.510920936239137, + "tokens_seen": 1525219328 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027167502507522567, + "loss": 2.7176, + "theoretical_loss": 3.5109076780143527, + "tokens_seen": 1525284864 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 842112, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0185139179229736, + "objective/train/theoretical_loss": 3.510894420518711, + "objective/train/tokens_used": 1545810400, + "theoretical_loss": 3.510894420518711, + "tokens_seen": 1525350400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716649949849549, + "loss": 2.7613, + "theoretical_loss": 3.510894420518711, + "tokens_seen": 1525350400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027165496489468403, + "loss": 2.7588, + "theoretical_loss": 3.51088116375214, + "tokens_seen": 1525415936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027164493480441327, + "loss": 2.7699, + "theoretical_loss": 3.510867907714569, + "tokens_seen": 1525481472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027163490471414245, + "loss": 2.648, + "theoretical_loss": 3.510854652405926, + "tokens_seen": 1525547008 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027162487462387163, + "loss": 2.7531, + "theoretical_loss": 3.51084139782614, + "tokens_seen": 1525612544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716148445336008, + "loss": 2.4952, + "theoretical_loss": 3.5108281439751394, + "tokens_seen": 1525678080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027160481444333, + "loss": 2.8022, + "theoretical_loss": 3.5108148908528527, + "tokens_seen": 1525743616 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715947843530592, + "loss": 2.7302, + "theoretical_loss": 3.5108016384592085, + "tokens_seen": 1525809152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715847542627884, + "loss": 2.514, + "theoretical_loss": 3.5107883867941356, + "tokens_seen": 1525874688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027157472417251754, + "loss": 2.5953, + "theoretical_loss": 3.5107751358575627, + "tokens_seen": 1525940224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715646940822468, + "loss": 2.6973, + "theoretical_loss": 3.5107618856494187, + "tokens_seen": 1526005760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715546639919759, + "loss": 2.582, + "theoretical_loss": 3.510748636169632, + "tokens_seen": 1526071296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027154463390170514, + "loss": 2.5885, + "theoretical_loss": 3.5107353874181313, + "tokens_seen": 1526136832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715346038114343, + "loss": 2.5398, + "theoretical_loss": 3.5107221393948453, + "tokens_seen": 1526202368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715245737211635, + "loss": 2.5864, + "theoretical_loss": 3.5107088920997027, + "tokens_seen": 1526267904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715145436308927, + "loss": 2.5209, + "theoretical_loss": 3.510695645532632, + "tokens_seen": 1526333440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715045135406219, + "loss": 2.4995, + "theoretical_loss": 3.5106823996935623, + "tokens_seen": 1526398976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027149448345035104, + "loss": 2.6928, + "theoretical_loss": 3.5106691545824225, + "tokens_seen": 1526464512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714844533600803, + "loss": 2.5935, + "theoretical_loss": 3.510655910199141, + "tokens_seen": 1526530048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714744232698094, + "loss": 2.3438, + "theoretical_loss": 3.510642666543646, + "tokens_seen": 1526595584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027146439317953864, + "loss": 2.5466, + "theoretical_loss": 3.510629423615867, + "tokens_seen": 1526661120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714543630892678, + "loss": 2.675, + "theoretical_loss": 3.510616181415733, + "tokens_seen": 1526726656 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271444332998997, + "loss": 2.5762, + "theoretical_loss": 3.510602939943172, + "tokens_seen": 1526792192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714343029087262, + "loss": 2.5912, + "theoretical_loss": 3.510589699198114, + "tokens_seen": 1526857728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027142427281845536, + "loss": 2.6199, + "theoretical_loss": 3.5105764591804864, + "tokens_seen": 1526923264 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 843396, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7498199939727783, + "objective/train/theoretical_loss": 3.510563219890219, + "objective/train/tokens_used": 1547448800, + "theoretical_loss": 3.510563219890219, + "tokens_seen": 1526988800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027141424272818454, + "loss": 2.7651, + "theoretical_loss": 3.510563219890219, + "tokens_seen": 1526988800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714042126379138, + "loss": 2.5227, + "theoretical_loss": 3.51054998132724, + "tokens_seen": 1527054336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713941825476429, + "loss": 2.4665, + "theoretical_loss": 3.5105367434914783, + "tokens_seen": 1527119872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027138415245737214, + "loss": 2.7011, + "theoretical_loss": 3.5105235063828633, + "tokens_seen": 1527185408 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027137412236710127, + "loss": 2.6465, + "theoretical_loss": 3.5105102700013235, + "tokens_seen": 1527250944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713640922768305, + "loss": 2.3794, + "theoretical_loss": 3.5104970343467876, + "tokens_seen": 1527316480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713540621865597, + "loss": 2.6201, + "theoretical_loss": 3.510483799419185, + "tokens_seen": 1527382016 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027134403209628887, + "loss": 2.5714, + "theoretical_loss": 3.510470565218444, + "tokens_seen": 1527447552 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027133400200601805, + "loss": 2.6446, + "theoretical_loss": 3.510457331744494, + "tokens_seen": 1527513088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713239719157473, + "loss": 2.6866, + "theoretical_loss": 3.5104440989972634, + "tokens_seen": 1527578624 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713139418254764, + "loss": 2.691, + "theoretical_loss": 3.510430866976681, + "tokens_seen": 1527644160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027130391173520564, + "loss": 2.6412, + "theoretical_loss": 3.5104176356826766, + "tokens_seen": 1527709696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027129388164493477, + "loss": 2.5747, + "theoretical_loss": 3.5104044051151786, + "tokens_seen": 1527775232 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271283851554664, + "loss": 2.6853, + "theoretical_loss": 3.5103911752741155, + "tokens_seen": 1527840768 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712738214643932, + "loss": 2.6371, + "theoretical_loss": 3.5103779461594176, + "tokens_seen": 1527906304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027126379137412237, + "loss": 2.7817, + "theoretical_loss": 3.5103647177710124, + "tokens_seen": 1527971840 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027125376128385155, + "loss": 2.7717, + "theoretical_loss": 3.5103514901088295, + "tokens_seen": 1528037376 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027124373119358073, + "loss": 2.4568, + "theoretical_loss": 3.510338263172798, + "tokens_seen": 1528102912 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712337011033099, + "loss": 2.8217, + "theoretical_loss": 3.5103250369628465, + "tokens_seen": 1528168448 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027122367101303915, + "loss": 2.6221, + "theoretical_loss": 3.5103118114789047, + "tokens_seen": 1528233984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712136409227683, + "loss": 2.372, + "theoretical_loss": 3.510298586720901, + "tokens_seen": 1528299520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712036108324975, + "loss": 2.5911, + "theoretical_loss": 3.5102853626887645, + "tokens_seen": 1528365056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711935807422267, + "loss": 2.6072, + "theoretical_loss": 3.5102721393824243, + "tokens_seen": 1528430592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027118355065195587, + "loss": 2.5907, + "theoretical_loss": 3.51025891680181, + "tokens_seen": 1528496128 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027117352056168505, + "loss": 2.8846, + "theoretical_loss": 3.5102456949468497, + "tokens_seen": 1528561664 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 844011, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.368853807449341, + "objective/train/theoretical_loss": 3.5102324738174726, + "objective/train/tokens_used": 1549087200, + "theoretical_loss": 3.5102324738174726, + "tokens_seen": 1528627200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027116349047141423, + "loss": 2.7747, + "theoretical_loss": 3.5102324738174726, + "tokens_seen": 1528627200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711534603811434, + "loss": 2.6397, + "theoretical_loss": 3.5102192534136085, + "tokens_seen": 1528692736 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027114343029087265, + "loss": 2.628, + "theoretical_loss": 3.5102060337351864, + "tokens_seen": 1528758272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711334002006018, + "loss": 2.4658, + "theoretical_loss": 3.5101928147821346, + "tokens_seen": 1528823808 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271123370110331, + "loss": 2.7907, + "theoretical_loss": 3.5101795965543827, + "tokens_seen": 1528889344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027111334002006014, + "loss": 2.6682, + "theoretical_loss": 3.51016637905186, + "tokens_seen": 1528954880 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711033099297894, + "loss": 2.6562, + "theoretical_loss": 3.5101531622744955, + "tokens_seen": 1529020416 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027109327983951856, + "loss": 2.4884, + "theoretical_loss": 3.510139946222218, + "tokens_seen": 1529085952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027108324974924774, + "loss": 2.6723, + "theoretical_loss": 3.510126730894957, + "tokens_seen": 1529151488 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710732196589769, + "loss": 2.5381, + "theoretical_loss": 3.5101135162926416, + "tokens_seen": 1529217024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710631895687061, + "loss": 2.6156, + "theoretical_loss": 3.510100302415201, + "tokens_seen": 1529282560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710531594784353, + "loss": 2.6253, + "theoretical_loss": 3.510087089262564, + "tokens_seen": 1529348096 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710431293881645, + "loss": 2.5449, + "theoretical_loss": 3.5100738768346607, + "tokens_seen": 1529413632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027103309929789364, + "loss": 2.6174, + "theoretical_loss": 3.5100606651314195, + "tokens_seen": 1529479168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710230692076229, + "loss": 2.845, + "theoretical_loss": 3.5100474541527693, + "tokens_seen": 1529544704 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710130391173521, + "loss": 2.6564, + "theoretical_loss": 3.51003424389864, + "tokens_seen": 1529610240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027100300902708124, + "loss": 2.6652, + "theoretical_loss": 3.510021034368961, + "tokens_seen": 1529675776 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709929789368105, + "loss": 2.6993, + "theoretical_loss": 3.510007825563661, + "tokens_seen": 1529741312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709829488465396, + "loss": 2.4723, + "theoretical_loss": 3.5099946174826693, + "tokens_seen": 1529806848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027097291875626884, + "loss": 2.6199, + "theoretical_loss": 3.5099814101259152, + "tokens_seen": 1529872384 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270962888665998, + "loss": 2.33, + "theoretical_loss": 3.5099682034933286, + "tokens_seen": 1529937920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709528585757272, + "loss": 2.6318, + "theoretical_loss": 3.5099549975848374, + "tokens_seen": 1530003456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709428284854564, + "loss": 2.8543, + "theoretical_loss": 3.5099417924003724, + "tokens_seen": 1530068992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027093279839518556, + "loss": 2.5853, + "theoretical_loss": 3.509928587939862, + "tokens_seen": 1530134528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027092276830491474, + "loss": 2.764, + "theoretical_loss": 3.509915384203236, + "tokens_seen": 1530200064 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 845259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.503072738647461, + "objective/train/theoretical_loss": 3.509902181190423, + "objective/train/tokens_used": 1550725600, + "theoretical_loss": 3.509902181190423, + "tokens_seen": 1530265600 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270912738214644, + "loss": 2.6888, + "theoretical_loss": 3.509902181190423, + "tokens_seen": 1530265600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709027081243731, + "loss": 2.3594, + "theoretical_loss": 3.5098889789013525, + "tokens_seen": 1530331136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027089267803410234, + "loss": 2.6613, + "theoretical_loss": 3.509875777335955, + "tokens_seen": 1530396672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027088264794383147, + "loss": 2.6888, + "theoretical_loss": 3.5098625764941582, + "tokens_seen": 1530462208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708726178535607, + "loss": 2.7254, + "theoretical_loss": 3.5098493763758922, + "tokens_seen": 1530527744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708625877632899, + "loss": 2.7197, + "theoretical_loss": 3.5098361769810866, + "tokens_seen": 1530593280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027085255767301907, + "loss": 2.5928, + "theoretical_loss": 3.5098229783096704, + "tokens_seen": 1530658816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027084252758274825, + "loss": 2.653, + "theoretical_loss": 3.509809780361573, + "tokens_seen": 1530724352 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708324974924775, + "loss": 2.5659, + "theoretical_loss": 3.509796583136724, + "tokens_seen": 1530789888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708224674022066, + "loss": 2.5242, + "theoretical_loss": 3.5097833866350525, + "tokens_seen": 1530855424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027081243731193584, + "loss": 2.5583, + "theoretical_loss": 3.5097701908564884, + "tokens_seen": 1530920960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027080240722166497, + "loss": 2.7689, + "theoretical_loss": 3.509756995800961, + "tokens_seen": 1530986496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707923771313942, + "loss": 2.7224, + "theoretical_loss": 3.5097438014683995, + "tokens_seen": 1531052032 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707823470411234, + "loss": 2.9064, + "theoretical_loss": 3.5097306078587334, + "tokens_seen": 1531117568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027077231695085257, + "loss": 2.5452, + "theoretical_loss": 3.5097174149718917, + "tokens_seen": 1531183104 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027076228686058175, + "loss": 2.727, + "theoretical_loss": 3.5097042228078053, + "tokens_seen": 1531248640 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027075225677031093, + "loss": 2.4024, + "theoretical_loss": 3.509691031366402, + "tokens_seen": 1531314176 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707422266800401, + "loss": 2.7561, + "theoretical_loss": 3.509677840647612, + "tokens_seen": 1531379712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027073219658976935, + "loss": 2.4852, + "theoretical_loss": 3.509664650651365, + "tokens_seen": 1531445248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707221664994985, + "loss": 2.8186, + "theoretical_loss": 3.50965146137759, + "tokens_seen": 1531510784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707121364092277, + "loss": 2.7045, + "theoretical_loss": 3.5096382728262174, + "tokens_seen": 1531576320 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707021063189569, + "loss": 2.5804, + "theoretical_loss": 3.509625084997176, + "tokens_seen": 1531641856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027069207622868607, + "loss": 2.8062, + "theoretical_loss": 3.5096118978903954, + "tokens_seen": 1531707392 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027068204613841525, + "loss": 2.5919, + "theoretical_loss": 3.509598711505805, + "tokens_seen": 1531772928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027067201604814443, + "loss": 2.3991, + "theoretical_loss": 3.5095855258433346, + "tokens_seen": 1531838464 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 846517, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4351906776428223, + "objective/train/theoretical_loss": 3.5095723409029143, + "objective/train/tokens_used": 1552364000, + "theoretical_loss": 3.5095723409029143, + "tokens_seen": 1531904000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706619859578736, + "loss": 2.5955, + "theoretical_loss": 3.5095723409029143, + "tokens_seen": 1531904000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027065195586760285, + "loss": 2.7506, + "theoretical_loss": 3.5095591566844724, + "tokens_seen": 1531969536 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270641925777332, + "loss": 2.6533, + "theoretical_loss": 3.5095459731879397, + "tokens_seen": 1532035072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706318956870612, + "loss": 2.6163, + "theoretical_loss": 3.5095327904132447, + "tokens_seen": 1532100608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027062186559679034, + "loss": 2.7896, + "theoretical_loss": 3.509519608360318, + "tokens_seen": 1532166144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706118355065196, + "loss": 2.5562, + "theoretical_loss": 3.5095064270290885, + "tokens_seen": 1532231680 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027060180541624876, + "loss": 2.6639, + "theoretical_loss": 3.5094932464194866, + "tokens_seen": 1532297216 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027059177532597794, + "loss": 2.336, + "theoretical_loss": 3.5094800665314416, + "tokens_seen": 1532362752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705817452357071, + "loss": 2.5906, + "theoretical_loss": 3.5094668873648827, + "tokens_seen": 1532428288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705717151454363, + "loss": 2.6533, + "theoretical_loss": 3.5094537089197395, + "tokens_seen": 1532493824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705616850551655, + "loss": 2.5801, + "theoretical_loss": 3.509440531195942, + "tokens_seen": 1532559360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705516549648947, + "loss": 2.6365, + "theoretical_loss": 3.5094273541934204, + "tokens_seen": 1532624896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027054162487462384, + "loss": 2.5615, + "theoretical_loss": 3.5094141779121037, + "tokens_seen": 1532690432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705315947843531, + "loss": 2.8698, + "theoretical_loss": 3.5094010023519218, + "tokens_seen": 1532755968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027052156469408226, + "loss": 2.6892, + "theoretical_loss": 3.5093878275128043, + "tokens_seen": 1532821504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027051153460381144, + "loss": 2.6264, + "theoretical_loss": 3.509374653394681, + "tokens_seen": 1532887040 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705015045135406, + "loss": 2.7703, + "theoretical_loss": 3.509361479997481, + "tokens_seen": 1532952576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704914744232698, + "loss": 2.79, + "theoretical_loss": 3.5093483073211353, + "tokens_seen": 1533018112 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270481444332999, + "loss": 2.7154, + "theoretical_loss": 3.509335135365573, + "tokens_seen": 1533083648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704714142427282, + "loss": 2.536, + "theoretical_loss": 3.5093219641307236, + "tokens_seen": 1533149184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027046138415245735, + "loss": 2.4691, + "theoretical_loss": 3.5093087936165173, + "tokens_seen": 1533214720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704513540621866, + "loss": 2.4994, + "theoretical_loss": 3.509295623822883, + "tokens_seen": 1533280256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704413239719157, + "loss": 2.7187, + "theoretical_loss": 3.5092824547497514, + "tokens_seen": 1533345792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027043129388164494, + "loss": 2.4823, + "theoretical_loss": 3.5092692863970525, + "tokens_seen": 1533411328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704212637913741, + "loss": 2.5204, + "theoretical_loss": 3.509256118764715, + "tokens_seen": 1533476864 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 847177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.847522735595703, + "objective/train/theoretical_loss": 3.50924295185267, + "objective/train/tokens_used": 1554002400, + "theoretical_loss": 3.50924295185267, + "tokens_seen": 1533542400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704112337011033, + "loss": 2.7513, + "theoretical_loss": 3.50924295185267, + "tokens_seen": 1533542400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704012036108325, + "loss": 2.4923, + "theoretical_loss": 3.509229785660846, + "tokens_seen": 1533607936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027039117352056167, + "loss": 2.5013, + "theoretical_loss": 3.5092166201891737, + "tokens_seen": 1533673472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027038114343029085, + "loss": 2.7783, + "theoretical_loss": 3.5092034554375826, + "tokens_seen": 1533739008 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703711133400201, + "loss": 2.7859, + "theoretical_loss": 3.5091902914060027, + "tokens_seen": 1533804544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703610832497492, + "loss": 2.5113, + "theoretical_loss": 3.509177128094364, + "tokens_seen": 1533870080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027035105315947845, + "loss": 2.6577, + "theoretical_loss": 3.509163965502596, + "tokens_seen": 1533935616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027034102306920763, + "loss": 2.4836, + "theoretical_loss": 3.509150803630629, + "tokens_seen": 1534001152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703309929789368, + "loss": 2.6504, + "theoretical_loss": 3.5091376424783927, + "tokens_seen": 1534066688 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270320962888666, + "loss": 2.6551, + "theoretical_loss": 3.5091244820458165, + "tokens_seen": 1534132224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027031093279839517, + "loss": 2.627, + "theoretical_loss": 3.5091113223328314, + "tokens_seen": 1534197760 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027030090270812435, + "loss": 2.5064, + "theoretical_loss": 3.5090981633393663, + "tokens_seen": 1534263296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002702908726178536, + "loss": 2.4783, + "theoretical_loss": 3.5090850050653515, + "tokens_seen": 1534328832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002702808425275827, + "loss": 2.4495, + "theoretical_loss": 3.509071847510717, + "tokens_seen": 1534394368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027027081243731195, + "loss": 2.6326, + "theoretical_loss": 3.509058690675393, + "tokens_seen": 1534459904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027026078234704113, + "loss": 2.6848, + "theoretical_loss": 3.5090455345593083, + "tokens_seen": 1534525440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702507522567703, + "loss": 2.6913, + "theoretical_loss": 3.5090323791623943, + "tokens_seen": 1534590976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027024072216649955, + "loss": 2.6024, + "theoretical_loss": 3.5090192244845806, + "tokens_seen": 1534656512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702306920762287, + "loss": 2.4758, + "theoretical_loss": 3.5090060705257966, + "tokens_seen": 1534722048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702206619859579, + "loss": 2.6506, + "theoretical_loss": 3.508992917285973, + "tokens_seen": 1534787584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702106318956871, + "loss": 2.5259, + "theoretical_loss": 3.5089797647650394, + "tokens_seen": 1534853120 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027020060180541627, + "loss": 2.5529, + "theoretical_loss": 3.508966612962926, + "tokens_seen": 1534918656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027019057171514545, + "loss": 2.537, + "theoretical_loss": 3.508953461879562, + "tokens_seen": 1534984192 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027018054162487463, + "loss": 2.718, + "theoretical_loss": 3.5089403115148787, + "tokens_seen": 1535049728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701705115346038, + "loss": 2.7283, + "theoretical_loss": 3.5089271618688054, + "tokens_seen": 1535115264 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 848526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7075657844543457, + "objective/train/theoretical_loss": 3.508914012941273, + "objective/train/tokens_used": 1555640800, + "theoretical_loss": 3.508914012941273, + "tokens_seen": 1535180800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027016048144433305, + "loss": 2.7353, + "theoretical_loss": 3.508914012941273, + "tokens_seen": 1535180800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701504513540622, + "loss": 2.609, + "theoretical_loss": 3.5089008647322104, + "tokens_seen": 1535246336 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701404212637914, + "loss": 2.5519, + "theoretical_loss": 3.5088877172415485, + "tokens_seen": 1535311872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027013039117352054, + "loss": 2.6782, + "theoretical_loss": 3.5088745704692164, + "tokens_seen": 1535377408 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701203610832498, + "loss": 2.1937, + "theoretical_loss": 3.5088614244151453, + "tokens_seen": 1535442944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027011033099297896, + "loss": 2.6903, + "theoretical_loss": 3.508848279079265, + "tokens_seen": 1535508480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027010030090270814, + "loss": 2.657, + "theoretical_loss": 3.5088351344615054, + "tokens_seen": 1535574016 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700902708124373, + "loss": 2.9042, + "theoretical_loss": 3.508821990561797, + "tokens_seen": 1535639552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700802407221665, + "loss": 2.7115, + "theoretical_loss": 3.5088088473800694, + "tokens_seen": 1535705088 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700702106318957, + "loss": 2.4773, + "theoretical_loss": 3.508795704916253, + "tokens_seen": 1535770624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700601805416249, + "loss": 2.801, + "theoretical_loss": 3.508782563170278, + "tokens_seen": 1535836160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027005015045135404, + "loss": 2.537, + "theoretical_loss": 3.5087694221420747, + "tokens_seen": 1535901696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700401203610833, + "loss": 2.8042, + "theoretical_loss": 3.5087562818315727, + "tokens_seen": 1535967232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027003009027081246, + "loss": 2.5515, + "theoretical_loss": 3.5087431422387025, + "tokens_seen": 1536032768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027002006018054164, + "loss": 2.5543, + "theoretical_loss": 3.508730003363395, + "tokens_seen": 1536098304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700100300902708, + "loss": 2.5299, + "theoretical_loss": 3.5087168652055793, + "tokens_seen": 1536163840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027, + "loss": 2.5026, + "theoretical_loss": 3.5087037277651865, + "tokens_seen": 1536229376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699899699097292, + "loss": 2.6706, + "theoretical_loss": 3.5086905910421455, + "tokens_seen": 1536294912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699799398194584, + "loss": 2.5769, + "theoretical_loss": 3.508677455036388, + "tokens_seen": 1536360448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026996990972918755, + "loss": 2.4922, + "theoretical_loss": 3.5086643197478438, + "tokens_seen": 1536425984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699598796389168, + "loss": 2.4464, + "theoretical_loss": 3.5086511851764426, + "tokens_seen": 1536491520 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699498495486459, + "loss": 2.7297, + "theoretical_loss": 3.508638051322116, + "tokens_seen": 1536557056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026993981945837514, + "loss": 2.6275, + "theoretical_loss": 3.5086249181847924, + "tokens_seen": 1536622592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699297893681043, + "loss": 2.5365, + "theoretical_loss": 3.508611785764403, + "tokens_seen": 1536688128 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699197592778335, + "loss": 2.6263, + "theoretical_loss": 3.5085986540608785, + "tokens_seen": 1536753664 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 849205, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6184380054473877, + "objective/train/theoretical_loss": 3.508585523074149, + "objective/train/tokens_used": 1557279200, + "theoretical_loss": 3.508585523074149, + "tokens_seen": 1536819200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699097291875627, + "loss": 2.833, + "theoretical_loss": 3.508585523074149, + "tokens_seen": 1536819200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026989969909729187, + "loss": 2.6451, + "theoretical_loss": 3.508572392804145, + "tokens_seen": 1536884736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026988966900702105, + "loss": 2.4547, + "theoretical_loss": 3.5085592632507954, + "tokens_seen": 1536950272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698796389167503, + "loss": 2.5255, + "theoretical_loss": 3.508546134414032, + "tokens_seen": 1537015808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698696088264794, + "loss": 2.4234, + "theoretical_loss": 3.508533006293785, + "tokens_seen": 1537081344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026985957873620865, + "loss": 2.6398, + "theoretical_loss": 3.5085198788899836, + "tokens_seen": 1537146880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026984954864593783, + "loss": 2.4771, + "theoretical_loss": 3.50850675220256, + "tokens_seen": 1537212416 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269839518555667, + "loss": 2.4766, + "theoretical_loss": 3.5084936262314432, + "tokens_seen": 1537277952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698294884653962, + "loss": 2.6984, + "theoretical_loss": 3.508480500976564, + "tokens_seen": 1537343488 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026981945837512537, + "loss": 2.5505, + "theoretical_loss": 3.508467376437853, + "tokens_seen": 1537409024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026980942828485455, + "loss": 2.6991, + "theoretical_loss": 3.50845425261524, + "tokens_seen": 1537474560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697993981945838, + "loss": 2.3392, + "theoretical_loss": 3.5084411295086557, + "tokens_seen": 1537540096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697893681043129, + "loss": 2.5386, + "theoretical_loss": 3.508428007118031, + "tokens_seen": 1537605632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026977933801404215, + "loss": 2.5687, + "theoretical_loss": 3.5084148854432957, + "tokens_seen": 1537671168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697693079237713, + "loss": 2.5538, + "theoretical_loss": 3.50840176448438, + "tokens_seen": 1537736704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697592778335005, + "loss": 2.4781, + "theoretical_loss": 3.508388644241215, + "tokens_seen": 1537802240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697492477432297, + "loss": 2.5457, + "theoretical_loss": 3.5083755247137316, + "tokens_seen": 1537867776 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697392176529589, + "loss": 2.509, + "theoretical_loss": 3.5083624059018588, + "tokens_seen": 1537933312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026972918756268806, + "loss": 2.6265, + "theoretical_loss": 3.5083492878055287, + "tokens_seen": 1537998848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697191574724173, + "loss": 2.4365, + "theoretical_loss": 3.50833617042467, + "tokens_seen": 1538064384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697091273821464, + "loss": 2.4423, + "theoretical_loss": 3.5083230537592147, + "tokens_seen": 1538129920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026969909729187565, + "loss": 2.5439, + "theoretical_loss": 3.5083099378090927, + "tokens_seen": 1538195456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696890672016048, + "loss": 2.6968, + "theoretical_loss": 3.5082968225742346, + "tokens_seen": 1538260992 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269679037111334, + "loss": 2.6952, + "theoretical_loss": 3.508283708054571, + "tokens_seen": 1538326528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696690070210632, + "loss": 2.488, + "theoretical_loss": 3.508270594250032, + "tokens_seen": 1538392064 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 849991, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9845397472381592, + "objective/train/theoretical_loss": 3.5082574811605487, + "objective/train/tokens_used": 1558917600, + "theoretical_loss": 3.5082574811605487, + "tokens_seen": 1538457600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696589769307924, + "loss": 2.4194, + "theoretical_loss": 3.5082574811605487, + "tokens_seen": 1538457600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026964894684052156, + "loss": 2.432, + "theoretical_loss": 3.5082443687860514, + "tokens_seen": 1538523136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026963891675025074, + "loss": 2.3859, + "theoretical_loss": 3.5082312571264707, + "tokens_seen": 1538588672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696288866599799, + "loss": 2.4426, + "theoretical_loss": 3.508218146181737, + "tokens_seen": 1538654208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026961885656970916, + "loss": 2.6011, + "theoretical_loss": 3.5082050359517813, + "tokens_seen": 1538719744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696088264794383, + "loss": 2.478, + "theoretical_loss": 3.5081919264365338, + "tokens_seen": 1538785280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695987963891675, + "loss": 2.5434, + "theoretical_loss": 3.508178817635925, + "tokens_seen": 1538850816 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026958876629889665, + "loss": 2.7497, + "theoretical_loss": 3.508165709549886, + "tokens_seen": 1538916352 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695787362086259, + "loss": 2.5912, + "theoretical_loss": 3.508152602178347, + "tokens_seen": 1538981888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026956870611835506, + "loss": 2.557, + "theoretical_loss": 3.508139495521239, + "tokens_seen": 1539047424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026955867602808424, + "loss": 2.5539, + "theoretical_loss": 3.5081263895784924, + "tokens_seen": 1539112960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695486459378134, + "loss": 2.2982, + "theoretical_loss": 3.508113284350038, + "tokens_seen": 1539178496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026953861584754266, + "loss": 2.8349, + "theoretical_loss": 3.5081001798358065, + "tokens_seen": 1539244032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695285857572718, + "loss": 2.7094, + "theoretical_loss": 3.5080870760357277, + "tokens_seen": 1539309568 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269518555667001, + "loss": 2.5591, + "theoretical_loss": 3.5080739729497337, + "tokens_seen": 1539375104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695085255767302, + "loss": 2.4333, + "theoretical_loss": 3.508060870577754, + "tokens_seen": 1539440640 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694984954864594, + "loss": 2.5765, + "theoretical_loss": 3.50804776891972, + "tokens_seen": 1539506176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694884653961886, + "loss": 2.84, + "theoretical_loss": 3.5080346679755623, + "tokens_seen": 1539571712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026947843530591775, + "loss": 2.3706, + "theoretical_loss": 3.508021567745211, + "tokens_seen": 1539637248 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269468405215647, + "loss": 2.644, + "theoretical_loss": 3.5080084682285975, + "tokens_seen": 1539702784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694583751253761, + "loss": 2.5932, + "theoretical_loss": 3.5079953694256525, + "tokens_seen": 1539768320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026944834503510534, + "loss": 2.496, + "theoretical_loss": 3.5079822713363065, + "tokens_seen": 1539833856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694383149448345, + "loss": 2.6042, + "theoretical_loss": 3.5079691739604906, + "tokens_seen": 1539899392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694282848545637, + "loss": 2.6964, + "theoretical_loss": 3.507956077298135, + "tokens_seen": 1539964928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694182547642929, + "loss": 2.7292, + "theoretical_loss": 3.5079429813491716, + "tokens_seen": 1540030464 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 850977, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2120277881622314, + "objective/train/theoretical_loss": 3.5079298861135295, + "objective/train/tokens_used": 1560556000, + "theoretical_loss": 3.5079298861135295, + "tokens_seen": 1540096000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026940822467402207, + "loss": 2.6, + "theoretical_loss": 3.5079298861135295, + "tokens_seen": 1540096000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026939819458375125, + "loss": 2.5619, + "theoretical_loss": 3.507916791591141, + "tokens_seen": 1540161536 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693881644934805, + "loss": 2.6881, + "theoretical_loss": 3.507903697781936, + "tokens_seen": 1540227072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693781344032096, + "loss": 2.5687, + "theoretical_loss": 3.507890604685845, + "tokens_seen": 1540292608 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026936810431293885, + "loss": 2.6343, + "theoretical_loss": 3.5078775123028003, + "tokens_seen": 1540358144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026935807422266803, + "loss": 2.7106, + "theoretical_loss": 3.507864420632732, + "tokens_seen": 1540423680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693480441323972, + "loss": 2.4071, + "theoretical_loss": 3.50785132967557, + "tokens_seen": 1540489216 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693380140421264, + "loss": 2.624, + "theoretical_loss": 3.5078382394312464, + "tokens_seen": 1540554752 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026932798395185557, + "loss": 2.5093, + "theoretical_loss": 3.5078251498996913, + "tokens_seen": 1540620288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026931795386158475, + "loss": 2.9694, + "theoretical_loss": 3.5078120610808363, + "tokens_seen": 1540685824 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269307923771314, + "loss": 2.6383, + "theoretical_loss": 3.5077989729746117, + "tokens_seen": 1540751360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692978936810431, + "loss": 2.7317, + "theoretical_loss": 3.5077858855809487, + "tokens_seen": 1540816896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026928786359077235, + "loss": 2.657, + "theoretical_loss": 3.5077727988997776, + "tokens_seen": 1540882432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692778335005015, + "loss": 2.4937, + "theoretical_loss": 3.5077597129310307, + "tokens_seen": 1540947968 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692678034102307, + "loss": 2.5744, + "theoretical_loss": 3.507746627674637, + "tokens_seen": 1541013504 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692577733199599, + "loss": 2.6681, + "theoretical_loss": 3.507733543130529, + "tokens_seen": 1541079040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692477432296891, + "loss": 2.5646, + "theoretical_loss": 3.5077204592986364, + "tokens_seen": 1541144576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026923771313941826, + "loss": 2.4769, + "theoretical_loss": 3.5077073761788915, + "tokens_seen": 1541210112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692276830491475, + "loss": 2.6539, + "theoretical_loss": 3.507694293771224, + "tokens_seen": 1541275648 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692176529588766, + "loss": 2.5781, + "theoretical_loss": 3.5076812120755663, + "tokens_seen": 1541341184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026920762286860585, + "loss": 2.6934, + "theoretical_loss": 3.507668131091848, + "tokens_seen": 1541406720 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269197592778335, + "loss": 2.8051, + "theoretical_loss": 3.5076550508200004, + "tokens_seen": 1541472256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691875626880642, + "loss": 2.6751, + "theoretical_loss": 3.5076419712599547, + "tokens_seen": 1541537792 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691775325977934, + "loss": 2.4307, + "theoretical_loss": 3.5076288924116423, + "tokens_seen": 1541603328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691675025075226, + "loss": 2.5462, + "theoretical_loss": 3.5076158142749936, + "tokens_seen": 1541668864 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 851654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2880167961120605, + "objective/train/theoretical_loss": 3.50760273684994, + "objective/train/tokens_used": 1562194400, + "theoretical_loss": 3.50760273684994, + "tokens_seen": 1541734400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026915747241725176, + "loss": 2.6604, + "theoretical_loss": 3.50760273684994, + "tokens_seen": 1541734400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026914744232698094, + "loss": 2.5, + "theoretical_loss": 3.5075896601364125, + "tokens_seen": 1541799936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691374122367101, + "loss": 2.7159, + "theoretical_loss": 3.5075765841343416, + "tokens_seen": 1541865472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026912738214643936, + "loss": 2.4402, + "theoretical_loss": 3.507563508843659, + "tokens_seen": 1541931008 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691173520561685, + "loss": 2.7302, + "theoretical_loss": 3.5075504342642954, + "tokens_seen": 1541996544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691073219658977, + "loss": 2.508, + "theoretical_loss": 3.5075373603961824, + "tokens_seen": 1542062080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026909729187562685, + "loss": 2.6313, + "theoretical_loss": 3.5075242872392502, + "tokens_seen": 1542127616 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690872617853561, + "loss": 2.4254, + "theoretical_loss": 3.5075112147934306, + "tokens_seen": 1542193152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026907723169508526, + "loss": 2.6111, + "theoretical_loss": 3.5074981430586547, + "tokens_seen": 1542258688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026906720160481444, + "loss": 2.4878, + "theoretical_loss": 3.5074850720348527, + "tokens_seen": 1542324224 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690571715145436, + "loss": 2.5306, + "theoretical_loss": 3.507472001721957, + "tokens_seen": 1542389760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026904714142427286, + "loss": 2.7115, + "theoretical_loss": 3.5074589321198983, + "tokens_seen": 1542455296 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269037111334002, + "loss": 2.6391, + "theoretical_loss": 3.5074458632286074, + "tokens_seen": 1542520832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690270812437312, + "loss": 2.7007, + "theoretical_loss": 3.5074327950480155, + "tokens_seen": 1542586368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026901705115346035, + "loss": 2.4498, + "theoretical_loss": 3.5074197275780543, + "tokens_seen": 1542651904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690070210631896, + "loss": 2.6167, + "theoretical_loss": 3.5074066608186545, + "tokens_seen": 1542717440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026899699097291877, + "loss": 2.4514, + "theoretical_loss": 3.5073935947697468, + "tokens_seen": 1542782976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026898696088264795, + "loss": 2.5525, + "theoretical_loss": 3.5073805294312637, + "tokens_seen": 1542848512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026897693079237713, + "loss": 2.4901, + "theoretical_loss": 3.507367464803135, + "tokens_seen": 1542914048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689669007021063, + "loss": 2.6087, + "theoretical_loss": 3.5073544008852933, + "tokens_seen": 1542979584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689568706118355, + "loss": 2.6454, + "theoretical_loss": 3.5073413376776683, + "tokens_seen": 1543045120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689468405215647, + "loss": 2.5948, + "theoretical_loss": 3.5073282751801926, + "tokens_seen": 1543110656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026893681043129385, + "loss": 2.6008, + "theoretical_loss": 3.5073152133927965, + "tokens_seen": 1543176192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689267803410231, + "loss": 2.6828, + "theoretical_loss": 3.507302152315412, + "tokens_seen": 1543241728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689167502507522, + "loss": 2.6454, + "theoretical_loss": 3.5072890919479693, + "tokens_seen": 1543307264 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 852770, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.775528907775879, + "objective/train/theoretical_loss": 3.5072760322904006, + "objective/train/tokens_used": 1563832800, + "theoretical_loss": 3.5072760322904006, + "tokens_seen": 1543372800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026890672016048145, + "loss": 2.7479, + "theoretical_loss": 3.5072760322904006, + "tokens_seen": 1543372800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026889669007021063, + "loss": 2.7586, + "theoretical_loss": 3.5072629733426375, + "tokens_seen": 1543438336 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688866599799398, + "loss": 2.6097, + "theoretical_loss": 3.5072499151046097, + "tokens_seen": 1543503872 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268876629889669, + "loss": 2.868, + "theoretical_loss": 3.50723685757625, + "tokens_seen": 1543569408 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026886659979939823, + "loss": 2.4749, + "theoretical_loss": 3.5072238007574894, + "tokens_seen": 1543634944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026885656970912736, + "loss": 2.523, + "theoretical_loss": 3.5072107446482583, + "tokens_seen": 1543700480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688465396188566, + "loss": 2.4207, + "theoretical_loss": 3.507197689248489, + "tokens_seen": 1543766016 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688365095285857, + "loss": 2.456, + "theoretical_loss": 3.5071846345581124, + "tokens_seen": 1543831552 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026882647943831495, + "loss": 2.3869, + "theoretical_loss": 3.5071715805770602, + "tokens_seen": 1543897088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026881644934804413, + "loss": 2.608, + "theoretical_loss": 3.507158527305264, + "tokens_seen": 1543962624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688064192577733, + "loss": 2.6021, + "theoretical_loss": 3.507145474742654, + "tokens_seen": 1544028160 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687963891675025, + "loss": 2.5998, + "theoretical_loss": 3.5071324228891623, + "tokens_seen": 1544093696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687863590772317, + "loss": 2.4491, + "theoretical_loss": 3.5071193717447207, + "tokens_seen": 1544159232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026877632898696086, + "loss": 2.6654, + "theoretical_loss": 3.5071063213092595, + "tokens_seen": 1544224768 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687662988966901, + "loss": 2.4587, + "theoretical_loss": 3.5070932715827112, + "tokens_seen": 1544290304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687562688064193, + "loss": 2.6008, + "theoretical_loss": 3.5070802225650066, + "tokens_seen": 1544355840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026874623871614846, + "loss": 2.3871, + "theoretical_loss": 3.5070671742560773, + "tokens_seen": 1544421376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687362086258777, + "loss": 2.6996, + "theoretical_loss": 3.507054126655855, + "tokens_seen": 1544486912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687261785356068, + "loss": 2.3082, + "theoretical_loss": 3.50704107976427, + "tokens_seen": 1544552448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026871614844533605, + "loss": 2.781, + "theoretical_loss": 3.507028033581255, + "tokens_seen": 1544617984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687061183550652, + "loss": 2.6394, + "theoretical_loss": 3.507014988106741, + "tokens_seen": 1544683520 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686960882647944, + "loss": 2.8959, + "theoretical_loss": 3.5070019433406596, + "tokens_seen": 1544749056 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686860581745236, + "loss": 2.5596, + "theoretical_loss": 3.506988899282942, + "tokens_seen": 1544814592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686760280842528, + "loss": 2.7267, + "theoretical_loss": 3.5069758559335202, + "tokens_seen": 1544880128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026866599799398196, + "loss": 2.437, + "theoretical_loss": 3.5069628132923256, + "tokens_seen": 1544945664 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 853806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.327728509902954, + "objective/train/theoretical_loss": 3.506949771359289, + "objective/train/tokens_used": 1565471200, + "theoretical_loss": 3.506949771359289, + "tokens_seen": 1545011200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026865596790371114, + "loss": 2.5891, + "theoretical_loss": 3.506949771359289, + "tokens_seen": 1545011200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686459378134403, + "loss": 2.6977, + "theoretical_loss": 3.5069367301343424, + "tokens_seen": 1545076736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026863590772316956, + "loss": 2.5861, + "theoretical_loss": 3.506923689617417, + "tokens_seen": 1545142272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686258776328987, + "loss": 2.8947, + "theoretical_loss": 3.5069106498084452, + "tokens_seen": 1545207808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686158475426279, + "loss": 2.6797, + "theoretical_loss": 3.506897610707358, + "tokens_seen": 1545273344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026860581745235705, + "loss": 2.5427, + "theoretical_loss": 3.5068845723140862, + "tokens_seen": 1545338880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685957873620863, + "loss": 2.569, + "theoretical_loss": 3.5068715346285626, + "tokens_seen": 1545404416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026858575727181546, + "loss": 2.6444, + "theoretical_loss": 3.506858497650718, + "tokens_seen": 1545469952 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026857572718154464, + "loss": 2.5422, + "theoretical_loss": 3.5068454613804843, + "tokens_seen": 1545535488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685656970912738, + "loss": 2.5319, + "theoretical_loss": 3.5068324258177936, + "tokens_seen": 1545601024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026855566700100306, + "loss": 2.5023, + "theoretical_loss": 3.506819390962576, + "tokens_seen": 1545666560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685456369107322, + "loss": 2.7061, + "theoretical_loss": 3.506806356814765, + "tokens_seen": 1545732096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685356068204614, + "loss": 2.6222, + "theoretical_loss": 3.5067933233742905, + "tokens_seen": 1545797632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026852557673019055, + "loss": 2.5943, + "theoretical_loss": 3.5067802906410854, + "tokens_seen": 1545863168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685155466399198, + "loss": 2.6763, + "theoretical_loss": 3.5067672586150804, + "tokens_seen": 1545928704 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026850551654964897, + "loss": 2.4709, + "theoretical_loss": 3.5067542272962076, + "tokens_seen": 1545994240 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026849548645937815, + "loss": 2.4995, + "theoretical_loss": 3.506741196684399, + "tokens_seen": 1546059776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026848545636910733, + "loss": 2.7153, + "theoretical_loss": 3.5067281667795855, + "tokens_seen": 1546125312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684754262788365, + "loss": 2.5729, + "theoretical_loss": 3.5067151375816996, + "tokens_seen": 1546190848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684653961885657, + "loss": 2.54, + "theoretical_loss": 3.5067021090906723, + "tokens_seen": 1546256384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684553660982949, + "loss": 2.6817, + "theoretical_loss": 3.506689081306435, + "tokens_seen": 1546321920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026844533600802405, + "loss": 2.5134, + "theoretical_loss": 3.5066760542289206, + "tokens_seen": 1546387456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684353059177533, + "loss": 2.3954, + "theoretical_loss": 3.5066630278580604, + "tokens_seen": 1546452992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684252758274824, + "loss": 2.5135, + "theoretical_loss": 3.506650002193785, + "tokens_seen": 1546518528 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026841524573721165, + "loss": 2.7969, + "theoretical_loss": 3.506636977236028, + "tokens_seen": 1546584064 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 854346, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0632946491241455, + "objective/train/theoretical_loss": 3.50662395298472, + "objective/train/tokens_used": 1567109600, + "theoretical_loss": 3.50662395298472, + "tokens_seen": 1546649600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026840521564694083, + "loss": 2.4863, + "theoretical_loss": 3.50662395298472, + "tokens_seen": 1546649600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026839518555667, + "loss": 2.6788, + "theoretical_loss": 3.506610929439792, + "tokens_seen": 1546715136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683851554663992, + "loss": 2.7006, + "theoretical_loss": 3.5065979066011774, + "tokens_seen": 1546780672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026837512537612843, + "loss": 2.6099, + "theoretical_loss": 3.5065848844688077, + "tokens_seen": 1546846208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026836509528585756, + "loss": 2.6737, + "theoretical_loss": 3.5065718630426135, + "tokens_seen": 1546911744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683550651955868, + "loss": 2.7256, + "theoretical_loss": 3.5065588423225273, + "tokens_seen": 1546977280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683450351053159, + "loss": 2.5789, + "theoretical_loss": 3.506545822308481, + "tokens_seen": 1547042816 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026833500501504515, + "loss": 2.5767, + "theoretical_loss": 3.5065328030004066, + "tokens_seen": 1547108352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026832497492477433, + "loss": 2.6697, + "theoretical_loss": 3.506519784398235, + "tokens_seen": 1547173888 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683149448345035, + "loss": 2.9253, + "theoretical_loss": 3.5065067665018996, + "tokens_seen": 1547239424 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683049147442327, + "loss": 2.7083, + "theoretical_loss": 3.506493749311331, + "tokens_seen": 1547304960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682948846539619, + "loss": 2.4743, + "theoretical_loss": 3.506480732826461, + "tokens_seen": 1547370496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026828485456369106, + "loss": 2.6113, + "theoretical_loss": 3.5064677170472223, + "tokens_seen": 1547436032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682748244734203, + "loss": 2.4489, + "theoretical_loss": 3.5064547019735457, + "tokens_seen": 1547501568 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682647943831494, + "loss": 2.6039, + "theoretical_loss": 3.5064416876053643, + "tokens_seen": 1547567104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026825476429287866, + "loss": 2.666, + "theoretical_loss": 3.506428673942609, + "tokens_seen": 1547632640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026825476429287866, + "loss": 2.5713, + "theoretical_loss": 3.506415660985212, + "tokens_seen": 1547698176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682447342026078, + "loss": 2.5002, + "theoretical_loss": 3.506402648733105, + "tokens_seen": 1547763712 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268234704112337, + "loss": 2.6364, + "theoretical_loss": 3.5063896371862207, + "tokens_seen": 1547829248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682246740220662, + "loss": 2.5491, + "theoretical_loss": 3.5063766263444895, + "tokens_seen": 1547894784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682146439317954, + "loss": 2.334, + "theoretical_loss": 3.506363616207845, + "tokens_seen": 1547960320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026820461384152456, + "loss": 2.5686, + "theoretical_loss": 3.506350606776219, + "tokens_seen": 1548025856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681945837512538, + "loss": 2.6391, + "theoretical_loss": 3.5063375980495417, + "tokens_seen": 1548091392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681845536609829, + "loss": 2.6821, + "theoretical_loss": 3.506324590027747, + "tokens_seen": 1548156928 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026817452357071216, + "loss": 2.8725, + "theoretical_loss": 3.506311582710766, + "tokens_seen": 1548222464 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 855711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7896533012390137, + "objective/train/theoretical_loss": 3.506298576098531, + "objective/train/tokens_used": 1568748000, + "theoretical_loss": 3.506298576098531, + "tokens_seen": 1548288000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681644934804413, + "loss": 2.6192, + "theoretical_loss": 3.506298576098531, + "tokens_seen": 1548288000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681544633901705, + "loss": 2.4303, + "theoretical_loss": 3.506285570190973, + "tokens_seen": 1548353536 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681444332998997, + "loss": 2.5779, + "theoretical_loss": 3.5062725649880253, + "tokens_seen": 1548419072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681344032096289, + "loss": 2.6168, + "theoretical_loss": 3.5062595604896196, + "tokens_seen": 1548484608 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026812437311935807, + "loss": 2.5535, + "theoretical_loss": 3.5062465566956877, + "tokens_seen": 1548550144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026811434302908725, + "loss": 2.5247, + "theoretical_loss": 3.506233553606161, + "tokens_seen": 1548615680 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026810431293881643, + "loss": 2.5174, + "theoretical_loss": 3.5062205512209728, + "tokens_seen": 1548681216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026809428284854566, + "loss": 2.5365, + "theoretical_loss": 3.506207549540054, + "tokens_seen": 1548746752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680842527582748, + "loss": 2.6277, + "theoretical_loss": 3.5061945485633377, + "tokens_seen": 1548812288 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268074222668004, + "loss": 2.4795, + "theoretical_loss": 3.5061815482907552, + "tokens_seen": 1548877824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026806419257773315, + "loss": 2.594, + "theoretical_loss": 3.506168548722239, + "tokens_seen": 1548943360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680541624874624, + "loss": 2.5768, + "theoretical_loss": 3.506155549857721, + "tokens_seen": 1549008896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026804413239719157, + "loss": 2.6516, + "theoretical_loss": 3.506142551697133, + "tokens_seen": 1549074432 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026803410230692075, + "loss": 2.5582, + "theoretical_loss": 3.506129554240407, + "tokens_seen": 1549139968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026802407221664993, + "loss": 2.7333, + "theoretical_loss": 3.506116557487476, + "tokens_seen": 1549205504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026801404212637917, + "loss": 2.6507, + "theoretical_loss": 3.5061035614382714, + "tokens_seen": 1549271040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026800401203610835, + "loss": 2.5752, + "theoretical_loss": 3.506090566092726, + "tokens_seen": 1549336576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026799398194583753, + "loss": 2.5158, + "theoretical_loss": 3.5060775714507715, + "tokens_seen": 1549402112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679839518555667, + "loss": 2.6866, + "theoretical_loss": 3.5060645775123396, + "tokens_seen": 1549467648 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679739217652959, + "loss": 2.6036, + "theoretical_loss": 3.5060515842773627, + "tokens_seen": 1549533184 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679638916750251, + "loss": 2.4638, + "theoretical_loss": 3.5060385917457735, + "tokens_seen": 1549598720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026795386158475425, + "loss": 2.7446, + "theoretical_loss": 3.506025599917504, + "tokens_seen": 1549664256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679438314944835, + "loss": 2.5557, + "theoretical_loss": 3.506012608792486, + "tokens_seen": 1549729792 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679338014042126, + "loss": 2.6441, + "theoretical_loss": 3.5059996183706517, + "tokens_seen": 1549795328 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026792377131394185, + "loss": 2.5275, + "theoretical_loss": 3.5059866286519337, + "tokens_seen": 1549860864 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 856331, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5446994304656982, + "objective/train/theoretical_loss": 3.505973639636264, + "objective/train/tokens_used": 1570386400, + "theoretical_loss": 3.505973639636264, + "tokens_seen": 1549926400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026791374122367103, + "loss": 2.5209, + "theoretical_loss": 3.505973639636264, + "tokens_seen": 1549926400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679037111334002, + "loss": 2.7458, + "theoretical_loss": 3.505960651323575, + "tokens_seen": 1549991936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678936810431294, + "loss": 2.6458, + "theoretical_loss": 3.5059476637137985, + "tokens_seen": 1550057472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026788365095285863, + "loss": 2.5634, + "theoretical_loss": 3.5059346768068673, + "tokens_seen": 1550123008 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026787362086258776, + "loss": 2.3974, + "theoretical_loss": 3.5059216906027135, + "tokens_seen": 1550188544 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267863590772317, + "loss": 2.2677, + "theoretical_loss": 3.505908705101269, + "tokens_seen": 1550254080 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678535606820461, + "loss": 2.5375, + "theoretical_loss": 3.5058957203024663, + "tokens_seen": 1550319616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026784353059177535, + "loss": 2.4942, + "theoretical_loss": 3.505882736206238, + "tokens_seen": 1550385152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026783350050150453, + "loss": 2.5636, + "theoretical_loss": 3.505869752812516, + "tokens_seen": 1550450688 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678234704112337, + "loss": 2.7162, + "theoretical_loss": 3.505856770121232, + "tokens_seen": 1550516224 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678134403209629, + "loss": 2.6569, + "theoretical_loss": 3.50584378813232, + "tokens_seen": 1550581760 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678034102306921, + "loss": 2.5652, + "theoretical_loss": 3.5058308068457107, + "tokens_seen": 1550647296 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026779338014042126, + "loss": 2.7396, + "theoretical_loss": 3.505817826261337, + "tokens_seen": 1550712832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677833500501505, + "loss": 2.7289, + "theoretical_loss": 3.5058048463791316, + "tokens_seen": 1550778368 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677733199598796, + "loss": 2.6986, + "theoretical_loss": 3.5057918671990267, + "tokens_seen": 1550843904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026776328986960886, + "loss": 2.6967, + "theoretical_loss": 3.505778888720954, + "tokens_seen": 1550909440 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267753259779338, + "loss": 2.6413, + "theoretical_loss": 3.5057659109448465, + "tokens_seen": 1550974976 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677432296890672, + "loss": 2.5232, + "theoretical_loss": 3.5057529338706366, + "tokens_seen": 1551040512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677331995987964, + "loss": 2.5715, + "theoretical_loss": 3.505739957498256, + "tokens_seen": 1551106048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677231695085256, + "loss": 2.5834, + "theoretical_loss": 3.505726981827638, + "tokens_seen": 1551171584 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026771313941825476, + "loss": 2.6938, + "theoretical_loss": 3.5057140068587147, + "tokens_seen": 1551237120 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267703109327984, + "loss": 2.5465, + "theoretical_loss": 3.5057010325914177, + "tokens_seen": 1551302656 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676930792377131, + "loss": 2.5455, + "theoretical_loss": 3.505688059025681, + "tokens_seen": 1551368192 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026768304914744236, + "loss": 2.7031, + "theoretical_loss": 3.505675086161436, + "tokens_seen": 1551433728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676730190571715, + "loss": 2.6595, + "theoretical_loss": 3.505662113998615, + "tokens_seen": 1551499264 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 857659, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6628549098968506, + "objective/train/theoretical_loss": 3.505649142537151, + "objective/train/tokens_used": 1572024800, + "theoretical_loss": 3.505649142537151, + "tokens_seen": 1551564800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676629889669007, + "loss": 2.6416, + "theoretical_loss": 3.505649142537151, + "tokens_seen": 1551564800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676529588766299, + "loss": 2.7571, + "theoretical_loss": 3.5056361717769757, + "tokens_seen": 1551630336 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676429287863591, + "loss": 2.6626, + "theoretical_loss": 3.5056232017180227, + "tokens_seen": 1551695872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026763289869608827, + "loss": 2.7074, + "theoretical_loss": 3.5056102323602234, + "tokens_seen": 1551761408 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026762286860581745, + "loss": 2.8588, + "theoretical_loss": 3.5055972637035104, + "tokens_seen": 1551826944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026761283851554663, + "loss": 2.4444, + "theoretical_loss": 3.505584295747817, + "tokens_seen": 1551892480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026760280842527586, + "loss": 2.5962, + "theoretical_loss": 3.5055713284930756, + "tokens_seen": 1551958016 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267592778335005, + "loss": 2.6036, + "theoretical_loss": 3.505558361939218, + "tokens_seen": 1552023552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675827482447342, + "loss": 2.5306, + "theoretical_loss": 3.505545396086177, + "tokens_seen": 1552089088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026757271815446335, + "loss": 2.6293, + "theoretical_loss": 3.5055324309338856, + "tokens_seen": 1552154624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675626880641926, + "loss": 2.692, + "theoretical_loss": 3.5055194664822755, + "tokens_seen": 1552220160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026755265797392177, + "loss": 2.6127, + "theoretical_loss": 3.5055065027312797, + "tokens_seen": 1552285696 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026754262788365095, + "loss": 2.6598, + "theoretical_loss": 3.505493539680831, + "tokens_seen": 1552351232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026753259779338013, + "loss": 2.4989, + "theoretical_loss": 3.5054805773308613, + "tokens_seen": 1552416768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026752256770310937, + "loss": 2.7468, + "theoretical_loss": 3.5054676156813036, + "tokens_seen": 1552482304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675125376128385, + "loss": 2.6931, + "theoretical_loss": 3.505454654732091, + "tokens_seen": 1552547840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026750250752256773, + "loss": 2.2726, + "theoretical_loss": 3.5054416944831557, + "tokens_seen": 1552613376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026749247743229686, + "loss": 2.5311, + "theoretical_loss": 3.50542873493443, + "tokens_seen": 1552678912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674824473420261, + "loss": 2.5356, + "theoretical_loss": 3.5054157760858464, + "tokens_seen": 1552744448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026747241725175527, + "loss": 2.5747, + "theoretical_loss": 3.505402817937338, + "tokens_seen": 1552809984 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026746238716148445, + "loss": 2.6245, + "theoretical_loss": 3.5053898604888376, + "tokens_seen": 1552875520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026745235707121363, + "loss": 2.5649, + "theoretical_loss": 3.5053769037402773, + "tokens_seen": 1552941056 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674423269809428, + "loss": 2.6746, + "theoretical_loss": 3.50536394769159, + "tokens_seen": 1553006592 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267432296890672, + "loss": 2.7146, + "theoretical_loss": 3.5053509923427084, + "tokens_seen": 1553072128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026742226680040123, + "loss": 2.6613, + "theoretical_loss": 3.505338037693565, + "tokens_seen": 1553137664 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 858740, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4517948627471924, + "objective/train/theoretical_loss": 3.5053250837440926, + "objective/train/tokens_used": 1573663200, + "theoretical_loss": 3.5053250837440926, + "tokens_seen": 1553203200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026741223671013036, + "loss": 2.7005, + "theoretical_loss": 3.5053250837440926, + "tokens_seen": 1553203200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674022066198596, + "loss": 2.5584, + "theoretical_loss": 3.505312130494224, + "tokens_seen": 1553268736 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673921765295888, + "loss": 2.6477, + "theoretical_loss": 3.5052991779438916, + "tokens_seen": 1553334272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026738214643931796, + "loss": 2.5397, + "theoretical_loss": 3.5052862260930286, + "tokens_seen": 1553399808 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026737211634904714, + "loss": 2.7688, + "theoretical_loss": 3.505273274941567, + "tokens_seen": 1553465344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673620862587763, + "loss": 2.6444, + "theoretical_loss": 3.5052603244894405, + "tokens_seen": 1553530880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673520561685055, + "loss": 2.7496, + "theoretical_loss": 3.505247374736581, + "tokens_seen": 1553596416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026734202607823474, + "loss": 2.5094, + "theoretical_loss": 3.5052344256829215, + "tokens_seen": 1553661952 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026733199598796386, + "loss": 2.682, + "theoretical_loss": 3.505221477328395, + "tokens_seen": 1553727488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673219658976931, + "loss": 2.7016, + "theoretical_loss": 3.5052085296729336, + "tokens_seen": 1553793024 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673119358074222, + "loss": 2.7309, + "theoretical_loss": 3.505195582716471, + "tokens_seen": 1553858560 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026730190571715146, + "loss": 2.4301, + "theoretical_loss": 3.5051826364589393, + "tokens_seen": 1553924096 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026729187562688064, + "loss": 2.4826, + "theoretical_loss": 3.5051696909002716, + "tokens_seen": 1553989632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672818455366098, + "loss": 2.5863, + "theoretical_loss": 3.5051567460404005, + "tokens_seen": 1554055168 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267271815446339, + "loss": 2.6842, + "theoretical_loss": 3.5051438018792593, + "tokens_seen": 1554120704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672617853560682, + "loss": 2.4586, + "theoretical_loss": 3.5051308584167806, + "tokens_seen": 1554186240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672517552657974, + "loss": 2.6992, + "theoretical_loss": 3.5051179156528964, + "tokens_seen": 1554251776 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672417251755266, + "loss": 2.6247, + "theoretical_loss": 3.5051049735875406, + "tokens_seen": 1554317312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672316950852558, + "loss": 2.5235, + "theoretical_loss": 3.5050920322206456, + "tokens_seen": 1554382848 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026722166499498496, + "loss": 2.4592, + "theoretical_loss": 3.5050790915521444, + "tokens_seen": 1554448384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672116349047142, + "loss": 2.7846, + "theoretical_loss": 3.50506615158197, + "tokens_seen": 1554513920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672016048144433, + "loss": 2.4958, + "theoretical_loss": 3.5050532123100546, + "tokens_seen": 1554579456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026719157472417256, + "loss": 2.8148, + "theoretical_loss": 3.5050402737363315, + "tokens_seen": 1554644992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671815446339017, + "loss": 2.7222, + "theoretical_loss": 3.5050273358607344, + "tokens_seen": 1554710528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671715145436309, + "loss": 2.5004, + "theoretical_loss": 3.505014398683195, + "tokens_seen": 1554776064 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 859255, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.550337314605713, + "objective/train/theoretical_loss": 3.505001462203647, + "objective/train/tokens_used": 1575301600, + "theoretical_loss": 3.505001462203647, + "tokens_seen": 1554841600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671614844533601, + "loss": 2.6438, + "theoretical_loss": 3.505001462203647, + "tokens_seen": 1554841600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671514543630893, + "loss": 2.6983, + "theoretical_loss": 3.5049885264220224, + "tokens_seen": 1554907136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026714142427281847, + "loss": 2.7964, + "theoretical_loss": 3.5049755913382548, + "tokens_seen": 1554972672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026713139418254765, + "loss": 2.6796, + "theoretical_loss": 3.504962656952278, + "tokens_seen": 1555038208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026712136409227683, + "loss": 2.4732, + "theoretical_loss": 3.504949723264023, + "tokens_seen": 1555103744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026711133400200606, + "loss": 2.6111, + "theoretical_loss": 3.5049367902734243, + "tokens_seen": 1555169280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671013039117352, + "loss": 2.6091, + "theoretical_loss": 3.5049238579804145, + "tokens_seen": 1555234816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002670912738214644, + "loss": 2.747, + "theoretical_loss": 3.504910926384926, + "tokens_seen": 1555300352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026708124373119355, + "loss": 2.6388, + "theoretical_loss": 3.5048979954868926, + "tokens_seen": 1555365888 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002670712136409228, + "loss": 2.6924, + "theoretical_loss": 3.504885065286247, + "tokens_seen": 1555431424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026706118355065197, + "loss": 2.5003, + "theoretical_loss": 3.5048721357829216, + "tokens_seen": 1555496960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026705115346038115, + "loss": 2.7032, + "theoretical_loss": 3.5048592069768505, + "tokens_seen": 1555562496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026704112337011033, + "loss": 2.7028, + "theoretical_loss": 3.5048462788679657, + "tokens_seen": 1555628032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026703109327983957, + "loss": 2.4155, + "theoretical_loss": 3.5048333514562016, + "tokens_seen": 1555693568 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002670210631895687, + "loss": 2.7196, + "theoretical_loss": 3.5048204247414896, + "tokens_seen": 1555759104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026701103309929793, + "loss": 2.7427, + "theoretical_loss": 3.5048074987237636, + "tokens_seen": 1555824640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026700100300902706, + "loss": 2.632, + "theoretical_loss": 3.5047945734029566, + "tokens_seen": 1555890176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669909729187563, + "loss": 2.6611, + "theoretical_loss": 3.504781648779002, + "tokens_seen": 1555955712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026698094282848547, + "loss": 2.6341, + "theoretical_loss": 3.504768724851832, + "tokens_seen": 1556021248 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026697091273821465, + "loss": 2.6446, + "theoretical_loss": 3.5047558016213807, + "tokens_seen": 1556086784 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026696088264794383, + "loss": 2.684, + "theoretical_loss": 3.5047428790875808, + "tokens_seen": 1556152320 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266950852557673, + "loss": 2.5949, + "theoretical_loss": 3.504729957250365, + "tokens_seen": 1556217856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669408224674022, + "loss": 2.5733, + "theoretical_loss": 3.5047170361096667, + "tokens_seen": 1556283392 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026693079237713143, + "loss": 2.6359, + "theoretical_loss": 3.5047041156654193, + "tokens_seen": 1556348928 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026692076228686056, + "loss": 2.283, + "theoretical_loss": 3.5046911959175553, + "tokens_seen": 1556414464 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 860458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.509343385696411, + "objective/train/theoretical_loss": 3.5046782768660085, + "objective/train/tokens_used": 1576940000, + "theoretical_loss": 3.5046782768660085, + "tokens_seen": 1556480000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669107321965898, + "loss": 2.5961, + "theoretical_loss": 3.5046782768660085, + "tokens_seen": 1556480000 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266900702106319, + "loss": 2.6762, + "theoretical_loss": 3.5046653585107124, + "tokens_seen": 1556545536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026689067201604816, + "loss": 2.3836, + "theoretical_loss": 3.504652440851599, + "tokens_seen": 1556611072 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026688064192577734, + "loss": 2.4974, + "theoretical_loss": 3.504639523888602, + "tokens_seen": 1556676608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668706118355065, + "loss": 2.412, + "theoretical_loss": 3.5046266076216552, + "tokens_seen": 1556742144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668605817452357, + "loss": 2.7806, + "theoretical_loss": 3.5046136920506905, + "tokens_seen": 1556807680 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026685055165496494, + "loss": 2.4857, + "theoretical_loss": 3.5046007771756424, + "tokens_seen": 1556873216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026684052156469406, + "loss": 2.5346, + "theoretical_loss": 3.504587862996443, + "tokens_seen": 1556938752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668304914744233, + "loss": 2.6091, + "theoretical_loss": 3.504574949513027, + "tokens_seen": 1557004288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668204613841524, + "loss": 2.6107, + "theoretical_loss": 3.504562036725326, + "tokens_seen": 1557069824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026681043129388166, + "loss": 2.4473, + "theoretical_loss": 3.504549124633274, + "tokens_seen": 1557135360 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026680040120361084, + "loss": 2.5327, + "theoretical_loss": 3.504536213236804, + "tokens_seen": 1557200896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026679037111334, + "loss": 2.6196, + "theoretical_loss": 3.5045233025358495, + "tokens_seen": 1557266432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667803410230692, + "loss": 2.6453, + "theoretical_loss": 3.504510392530344, + "tokens_seen": 1557331968 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667703109327984, + "loss": 2.6411, + "theoretical_loss": 3.5044974832202205, + "tokens_seen": 1557397504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026676028084252757, + "loss": 2.6207, + "theoretical_loss": 3.504484574605412, + "tokens_seen": 1557463040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667502507522568, + "loss": 2.5174, + "theoretical_loss": 3.5044716666858524, + "tokens_seen": 1557528576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026674022066198593, + "loss": 2.6171, + "theoretical_loss": 3.5044587594614742, + "tokens_seen": 1557594112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026673019057171516, + "loss": 2.7117, + "theoretical_loss": 3.5044458529322116, + "tokens_seen": 1557659648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026672016048144434, + "loss": 2.6978, + "theoretical_loss": 3.5044329470979974, + "tokens_seen": 1557725184 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667101303911735, + "loss": 2.5189, + "theoretical_loss": 3.504420041958765, + "tokens_seen": 1557790720 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667001003009027, + "loss": 2.3499, + "theoretical_loss": 3.5044071375144474, + "tokens_seen": 1557856256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666900702106319, + "loss": 2.4713, + "theoretical_loss": 3.5043942337649785, + "tokens_seen": 1557921792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026668004012036107, + "loss": 2.7262, + "theoretical_loss": 3.5043813307102916, + "tokens_seen": 1557987328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666700100300903, + "loss": 2.7239, + "theoretical_loss": 3.50436842835032, + "tokens_seen": 1558052864 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 860986, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4487876892089844, + "objective/train/theoretical_loss": 3.5043555266849973, + "objective/train/tokens_used": 1578578400, + "theoretical_loss": 3.5043555266849973, + "tokens_seen": 1558118400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026665997993981943, + "loss": 2.4259, + "theoretical_loss": 3.5043555266849973, + "tokens_seen": 1558118400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026664994984954867, + "loss": 2.698, + "theoretical_loss": 3.504342625714256, + "tokens_seen": 1558183936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666399197592778, + "loss": 2.7367, + "theoretical_loss": 3.5043297254380303, + "tokens_seen": 1558249472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026662988966900703, + "loss": 2.613, + "theoretical_loss": 3.5043168258562534, + "tokens_seen": 1558315008 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666198595787362, + "loss": 2.4404, + "theoretical_loss": 3.5043039269688587, + "tokens_seen": 1558380544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666098294884654, + "loss": 2.3962, + "theoretical_loss": 3.50429102877578, + "tokens_seen": 1558446080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026659979939819457, + "loss": 2.7861, + "theoretical_loss": 3.5042781312769495, + "tokens_seen": 1558511616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026658976930792375, + "loss": 2.5324, + "theoretical_loss": 3.504265234472302, + "tokens_seen": 1558577152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026657973921765293, + "loss": 2.5691, + "theoretical_loss": 3.5042523383617707, + "tokens_seen": 1558642688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026656970912738217, + "loss": 2.678, + "theoretical_loss": 3.5042394429452886, + "tokens_seen": 1558708224 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665596790371113, + "loss": 2.7404, + "theoretical_loss": 3.50422654822279, + "tokens_seen": 1558773760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026654964894684053, + "loss": 2.411, + "theoretical_loss": 3.504213654194207, + "tokens_seen": 1558839296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665396188565697, + "loss": 2.6935, + "theoretical_loss": 3.504200760859474, + "tokens_seen": 1558904832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665295887662989, + "loss": 2.4234, + "theoretical_loss": 3.504187868218524, + "tokens_seen": 1558970368 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665195586760281, + "loss": 2.7451, + "theoretical_loss": 3.5041749762712913, + "tokens_seen": 1559035904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026650952858575726, + "loss": 2.6332, + "theoretical_loss": 3.5041620850177093, + "tokens_seen": 1559101440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664994984954865, + "loss": 2.564, + "theoretical_loss": 3.5041491944577103, + "tokens_seen": 1559166976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026648946840521567, + "loss": 2.8275, + "theoretical_loss": 3.5041363045912295, + "tokens_seen": 1559232512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026647943831494485, + "loss": 2.4974, + "theoretical_loss": 3.5041234154181993, + "tokens_seen": 1559298048 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026646940822467403, + "loss": 2.5962, + "theoretical_loss": 3.504110526938554, + "tokens_seen": 1559363584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664593781344032, + "loss": 2.7282, + "theoretical_loss": 3.504097639152226, + "tokens_seen": 1559429120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664493480441324, + "loss": 2.4502, + "theoretical_loss": 3.5040847520591503, + "tokens_seen": 1559494656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026643931795386163, + "loss": 2.7018, + "theoretical_loss": 3.5040718656592595, + "tokens_seen": 1559560192 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026642928786359076, + "loss": 2.4882, + "theoretical_loss": 3.5040589799524877, + "tokens_seen": 1559625728 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026641925777332, + "loss": 2.5176, + "theoretical_loss": 3.504046094938768, + "tokens_seen": 1559691264 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 862298, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.92509126663208, + "objective/train/theoretical_loss": 3.5040332106180347, + "objective/train/tokens_used": 1580216800, + "theoretical_loss": 3.5040332106180347, + "tokens_seen": 1559756800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664092276830492, + "loss": 2.678, + "theoretical_loss": 3.5040332106180347, + "tokens_seen": 1559756800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026639919759277836, + "loss": 2.6686, + "theoretical_loss": 3.5040203269902213, + "tokens_seen": 1559822336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026638916750250754, + "loss": 2.4537, + "theoretical_loss": 3.5040074440552607, + "tokens_seen": 1559887872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663791374122367, + "loss": 2.575, + "theoretical_loss": 3.5039945618130863, + "tokens_seen": 1559953408 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663691073219659, + "loss": 2.6144, + "theoretical_loss": 3.5039816802636334, + "tokens_seen": 1560018944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026635907723169514, + "loss": 2.5371, + "theoretical_loss": 3.5039687994068345, + "tokens_seen": 1560084480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026634904714142426, + "loss": 2.5255, + "theoretical_loss": 3.5039559192426233, + "tokens_seen": 1560150016 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663390170511535, + "loss": 2.7876, + "theoretical_loss": 3.5039430397709337, + "tokens_seen": 1560215552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663289869608826, + "loss": 2.7222, + "theoretical_loss": 3.503930160991699, + "tokens_seen": 1560281088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026631895687061186, + "loss": 2.4917, + "theoretical_loss": 3.5039172829048537, + "tokens_seen": 1560346624 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026630892678034104, + "loss": 2.6684, + "theoretical_loss": 3.5039044055103306, + "tokens_seen": 1560412160 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662988966900702, + "loss": 2.6276, + "theoretical_loss": 3.503891528808064, + "tokens_seen": 1560477696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662888665997994, + "loss": 2.5074, + "theoretical_loss": 3.5038786527979866, + "tokens_seen": 1560543232 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662788365095286, + "loss": 2.555, + "theoretical_loss": 3.503865777480034, + "tokens_seen": 1560608768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026626880641925777, + "loss": 2.551, + "theoretical_loss": 3.5038529028541383, + "tokens_seen": 1560674304 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266258776328987, + "loss": 2.6477, + "theoretical_loss": 3.503840028920234, + "tokens_seen": 1560739840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026624874623871613, + "loss": 2.6122, + "theoretical_loss": 3.5038271556782545, + "tokens_seen": 1560805376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026623871614844536, + "loss": 2.7329, + "theoretical_loss": 3.5038142831281336, + "tokens_seen": 1560870912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026622868605817454, + "loss": 2.9113, + "theoretical_loss": 3.5038014112698055, + "tokens_seen": 1560936448 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662186559679037, + "loss": 2.4849, + "theoretical_loss": 3.503788540103203, + "tokens_seen": 1561001984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662086258776329, + "loss": 2.5958, + "theoretical_loss": 3.5037756696282614, + "tokens_seen": 1561067520 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661985957873621, + "loss": 2.631, + "theoretical_loss": 3.503762799844913, + "tokens_seen": 1561133056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026618856569709127, + "loss": 2.7846, + "theoretical_loss": 3.503749930753093, + "tokens_seen": 1561198592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661785356068205, + "loss": 2.5079, + "theoretical_loss": 3.5037370623527337, + "tokens_seen": 1561264128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026616850551654963, + "loss": 2.5375, + "theoretical_loss": 3.50372419464377, + "tokens_seen": 1561329664 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 862728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6292545795440674, + "objective/train/theoretical_loss": 3.503711327626135, + "objective/train/tokens_used": 1581855200, + "theoretical_loss": 3.503711327626135, + "tokens_seen": 1561395200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026615847542627887, + "loss": 2.7543, + "theoretical_loss": 3.503711327626135, + "tokens_seen": 1561395200 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266148445336008, + "loss": 2.6994, + "theoretical_loss": 3.5036984612997637, + "tokens_seen": 1561460736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026613841524573723, + "loss": 2.5635, + "theoretical_loss": 3.5036855956645887, + "tokens_seen": 1561526272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661283851554664, + "loss": 2.6385, + "theoretical_loss": 3.5036727307205444, + "tokens_seen": 1561591808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661183550651956, + "loss": 2.6418, + "theoretical_loss": 3.503659866467565, + "tokens_seen": 1561657344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026610832497492477, + "loss": 2.6659, + "theoretical_loss": 3.503647002905584, + "tokens_seen": 1561722880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026609829488465395, + "loss": 2.4739, + "theoretical_loss": 3.503634140034535, + "tokens_seen": 1561788416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026608826479438313, + "loss": 2.8646, + "theoretical_loss": 3.5036212778543527, + "tokens_seen": 1561853952 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026607823470411237, + "loss": 2.7653, + "theoretical_loss": 3.50360841636497, + "tokens_seen": 1561919488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660682046138415, + "loss": 2.6995, + "theoretical_loss": 3.5035955555663216, + "tokens_seen": 1561985024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026605817452357073, + "loss": 2.6088, + "theoretical_loss": 3.5035826954583413, + "tokens_seen": 1562050560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660481444332999, + "loss": 2.5755, + "theoretical_loss": 3.5035698360409624, + "tokens_seen": 1562116096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660381143430291, + "loss": 2.6099, + "theoretical_loss": 3.50355697731412, + "tokens_seen": 1562181632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660280842527583, + "loss": 2.6855, + "theoretical_loss": 3.5035441192777466, + "tokens_seen": 1562247168 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026601805416248746, + "loss": 2.8502, + "theoretical_loss": 3.5035312619317773, + "tokens_seen": 1562312704 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026600802407221664, + "loss": 2.4822, + "theoretical_loss": 3.503518405276146, + "tokens_seen": 1562378240 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026599799398194587, + "loss": 2.5384, + "theoretical_loss": 3.503505549310786, + "tokens_seen": 1562443776 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265987963891675, + "loss": 2.5567, + "theoretical_loss": 3.503492694035632, + "tokens_seen": 1562509312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026597793380140423, + "loss": 2.6505, + "theoretical_loss": 3.503479839450618, + "tokens_seen": 1562574848 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026596790371113336, + "loss": 2.4879, + "theoretical_loss": 3.503466985555677, + "tokens_seen": 1562640384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659578736208626, + "loss": 2.5259, + "theoretical_loss": 3.503454132350744, + "tokens_seen": 1562705920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659478435305918, + "loss": 2.6594, + "theoretical_loss": 3.503441279835753, + "tokens_seen": 1562771456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026593781344032096, + "loss": 2.561, + "theoretical_loss": 3.503428428010637, + "tokens_seen": 1562836992 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026592778335005014, + "loss": 2.6891, + "theoretical_loss": 3.5034155768753314, + "tokens_seen": 1562902528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659177532597794, + "loss": 2.5436, + "theoretical_loss": 3.5034027264297696, + "tokens_seen": 1562968064 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 863704, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2020814418792725, + "objective/train/theoretical_loss": 3.503389876673886, + "objective/train/tokens_used": 1583493600, + "theoretical_loss": 3.503389876673886, + "tokens_seen": 1563033600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659077231695085, + "loss": 2.7556, + "theoretical_loss": 3.503389876673886, + "tokens_seen": 1563033600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026589769307923774, + "loss": 2.6175, + "theoretical_loss": 3.503377027607614, + "tokens_seen": 1563099136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026588766298896686, + "loss": 2.6476, + "theoretical_loss": 3.5033641792308883, + "tokens_seen": 1563164672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658776328986961, + "loss": 2.6468, + "theoretical_loss": 3.5033513315436426, + "tokens_seen": 1563230208 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658676028084253, + "loss": 2.5511, + "theoretical_loss": 3.5033384845458113, + "tokens_seen": 1563295744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026585757271815446, + "loss": 2.7945, + "theoretical_loss": 3.503325638237328, + "tokens_seen": 1563361280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026584754262788364, + "loss": 2.6927, + "theoretical_loss": 3.503312792618128, + "tokens_seen": 1563426816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658375125376128, + "loss": 2.559, + "theoretical_loss": 3.5032999476881437, + "tokens_seen": 1563492352 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265827482447342, + "loss": 2.557, + "theoretical_loss": 3.5032871034473105, + "tokens_seen": 1563557888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026581745235707124, + "loss": 2.6794, + "theoretical_loss": 3.503274259895562, + "tokens_seen": 1563623424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026580742226680037, + "loss": 2.4775, + "theoretical_loss": 3.5032614170328333, + "tokens_seen": 1563688960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657973921765296, + "loss": 2.5448, + "theoretical_loss": 3.5032485748590574, + "tokens_seen": 1563754496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026578736208625873, + "loss": 2.5522, + "theoretical_loss": 3.5032357333741686, + "tokens_seen": 1563820032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026577733199598797, + "loss": 2.5624, + "theoretical_loss": 3.5032228925781013, + "tokens_seen": 1563885568 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026576730190571715, + "loss": 2.5355, + "theoretical_loss": 3.50321005247079, + "tokens_seen": 1563951104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026575727181544633, + "loss": 2.6652, + "theoretical_loss": 3.503197213052169, + "tokens_seen": 1564016640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026574724172517556, + "loss": 2.6279, + "theoretical_loss": 3.5031843743221716, + "tokens_seen": 1564082176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026573721163490474, + "loss": 2.5509, + "theoretical_loss": 3.5031715362807327, + "tokens_seen": 1564147712 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657271815446339, + "loss": 2.4891, + "theoretical_loss": 3.503158698927786, + "tokens_seen": 1564213248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657171514543631, + "loss": 2.5214, + "theoretical_loss": 3.5031458622632665, + "tokens_seen": 1564278784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657071213640923, + "loss": 2.3911, + "theoretical_loss": 3.503133026287108, + "tokens_seen": 1564344320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026569709127382147, + "loss": 2.5955, + "theoretical_loss": 3.503120190999245, + "tokens_seen": 1564409856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656870611835507, + "loss": 2.5199, + "theoretical_loss": 3.503107356399611, + "tokens_seen": 1564475392 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026567703109327983, + "loss": 2.4505, + "theoretical_loss": 3.5030945224881416, + "tokens_seen": 1564540928 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026566700100300907, + "loss": 2.6076, + "theoretical_loss": 3.5030816892647696, + "tokens_seen": 1564606464 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 864198, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.547630548477173, + "objective/train/theoretical_loss": 3.5030688567294304, + "objective/train/tokens_used": 1585132000, + "theoretical_loss": 3.5030688567294304, + "tokens_seen": 1564672000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656569709127382, + "loss": 2.6312, + "theoretical_loss": 3.5030688567294304, + "tokens_seen": 1564672000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026564694082246743, + "loss": 2.3489, + "theoretical_loss": 3.503056024882058, + "tokens_seen": 1564737536 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656369107321966, + "loss": 2.605, + "theoretical_loss": 3.5030431937225863, + "tokens_seen": 1564803072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656268806419258, + "loss": 2.5861, + "theoretical_loss": 3.50303036325095, + "tokens_seen": 1564868608 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026561685055165497, + "loss": 2.6797, + "theoretical_loss": 3.5030175334670837, + "tokens_seen": 1564934144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026560682046138415, + "loss": 2.5567, + "theoretical_loss": 3.503004704370921, + "tokens_seen": 1564999680 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026559679037111333, + "loss": 2.4033, + "theoretical_loss": 3.5029918759623966, + "tokens_seen": 1565065216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026558676028084257, + "loss": 2.67, + "theoretical_loss": 3.5029790482414453, + "tokens_seen": 1565130752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655767301905717, + "loss": 2.693, + "theoretical_loss": 3.502966221208, + "tokens_seen": 1565196288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026556670010030093, + "loss": 2.4977, + "theoretical_loss": 3.502953394861997, + "tokens_seen": 1565261824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655566700100301, + "loss": 2.749, + "theoretical_loss": 3.50294056920337, + "tokens_seen": 1565327360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655466399197593, + "loss": 2.2335, + "theoretical_loss": 3.5029277442320526, + "tokens_seen": 1565392896 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655366098294885, + "loss": 2.4006, + "theoretical_loss": 3.5029149199479797, + "tokens_seen": 1565458432 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026552657973921766, + "loss": 2.5632, + "theoretical_loss": 3.502902096351086, + "tokens_seen": 1565523968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026551654964894684, + "loss": 2.7731, + "theoretical_loss": 3.5028892734413057, + "tokens_seen": 1565589504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026550651955867607, + "loss": 2.5731, + "theoretical_loss": 3.5028764512185733, + "tokens_seen": 1565655040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654964894684052, + "loss": 2.5059, + "theoretical_loss": 3.502863629682823, + "tokens_seen": 1565720576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026548645937813443, + "loss": 2.6426, + "theoretical_loss": 3.5028508088339887, + "tokens_seen": 1565786112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026547642928786356, + "loss": 2.5619, + "theoretical_loss": 3.5028379886720065, + "tokens_seen": 1565851648 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654663991975928, + "loss": 2.3653, + "theoretical_loss": 3.5028251691968095, + "tokens_seen": 1565917184 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265456369107322, + "loss": 2.75, + "theoretical_loss": 3.5028123504083326, + "tokens_seen": 1565982720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026544633901705116, + "loss": 2.5192, + "theoretical_loss": 3.5027995323065104, + "tokens_seen": 1566048256 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026543630892678034, + "loss": 2.3786, + "theoretical_loss": 3.502786714891277, + "tokens_seen": 1566113792 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654262788365096, + "loss": 2.7013, + "theoretical_loss": 3.502773898162567, + "tokens_seen": 1566179328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654162487462387, + "loss": 2.5743, + "theoretical_loss": 3.5027610821203146, + "tokens_seen": 1566244864 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 865502, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5836033821105957, + "objective/train/theoretical_loss": 3.5027482667644554, + "objective/train/tokens_used": 1586770400, + "theoretical_loss": 3.5027482667644554, + "tokens_seen": 1566310400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026540621865596794, + "loss": 2.6275, + "theoretical_loss": 3.5027482667644554, + "tokens_seen": 1566310400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026539618856569707, + "loss": 2.5517, + "theoretical_loss": 3.5027354520949228, + "tokens_seen": 1566375936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653861584754263, + "loss": 2.6707, + "theoretical_loss": 3.5027226381116523, + "tokens_seen": 1566441472 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653761283851555, + "loss": 2.64, + "theoretical_loss": 3.502709824814578, + "tokens_seen": 1566507008 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026536609829488466, + "loss": 2.3741, + "theoretical_loss": 3.5026970122036336, + "tokens_seen": 1566572544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026535606820461384, + "loss": 2.5363, + "theoretical_loss": 3.502684200278755, + "tokens_seen": 1566638080 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265346038114343, + "loss": 2.5694, + "theoretical_loss": 3.5026713890398757, + "tokens_seen": 1566703616 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653360080240722, + "loss": 2.6299, + "theoretical_loss": 3.502658578486931, + "tokens_seen": 1566769152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026532597793380144, + "loss": 2.5698, + "theoretical_loss": 3.5026457686198555, + "tokens_seen": 1566834688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026531594784353057, + "loss": 2.6487, + "theoretical_loss": 3.5026329594385834, + "tokens_seen": 1566900224 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653059177532598, + "loss": 2.5744, + "theoretical_loss": 3.5026201509430495, + "tokens_seen": 1566965760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026529588766298893, + "loss": 2.6302, + "theoretical_loss": 3.502607343133188, + "tokens_seen": 1567031296 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026528585757271817, + "loss": 2.5629, + "theoretical_loss": 3.5025945360089343, + "tokens_seen": 1567096832 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026527582748244735, + "loss": 2.5357, + "theoretical_loss": 3.502581729570222, + "tokens_seen": 1567162368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026526579739217653, + "loss": 2.5799, + "theoretical_loss": 3.5025689238169866, + "tokens_seen": 1567227904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652557673019057, + "loss": 2.5381, + "theoretical_loss": 3.5025561187491627, + "tokens_seen": 1567293440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026524573721163494, + "loss": 2.4347, + "theoretical_loss": 3.502543314366685, + "tokens_seen": 1567358976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026523570712136407, + "loss": 2.5106, + "theoretical_loss": 3.5025305106694873, + "tokens_seen": 1567424512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652256770310933, + "loss": 2.4497, + "theoretical_loss": 3.502517707657505, + "tokens_seen": 1567490048 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026521564694082243, + "loss": 2.4718, + "theoretical_loss": 3.5025049053306727, + "tokens_seen": 1567555584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026520561685055167, + "loss": 2.6144, + "theoretical_loss": 3.502492103688925, + "tokens_seen": 1567621120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026519558676028085, + "loss": 2.6231, + "theoretical_loss": 3.502479302732197, + "tokens_seen": 1567686656 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026518555667001003, + "loss": 2.3729, + "theoretical_loss": 3.502466502460423, + "tokens_seen": 1567752192 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651755265797392, + "loss": 2.6039, + "theoretical_loss": 3.502453702873538, + "tokens_seen": 1567817728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651654964894684, + "loss": 2.5398, + "theoretical_loss": 3.502440903971476, + "tokens_seen": 1567883264 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 866057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.337247371673584, + "objective/train/theoretical_loss": 3.5024281057541726, + "objective/train/tokens_used": 1588408800, + "theoretical_loss": 3.5024281057541726, + "tokens_seen": 1567948800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651554663991976, + "loss": 2.5498, + "theoretical_loss": 3.5024281057541726, + "tokens_seen": 1567948800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651454363089268, + "loss": 2.7671, + "theoretical_loss": 3.502415308221562, + "tokens_seen": 1568014336 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026513540621865594, + "loss": 2.5185, + "theoretical_loss": 3.502402511373579, + "tokens_seen": 1568079872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026512537612838517, + "loss": 2.528, + "theoretical_loss": 3.5023897152101586, + "tokens_seen": 1568145408 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651153460381143, + "loss": 2.4021, + "theoretical_loss": 3.502376919731236, + "tokens_seen": 1568210944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026510531594784353, + "loss": 2.5057, + "theoretical_loss": 3.5023641249367445, + "tokens_seen": 1568276480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650952858575727, + "loss": 2.653, + "theoretical_loss": 3.5023513308266203, + "tokens_seen": 1568342016 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650852557673019, + "loss": 2.7023, + "theoretical_loss": 3.5023385374007985, + "tokens_seen": 1568407552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650752256770311, + "loss": 2.4164, + "theoretical_loss": 3.5023257446592124, + "tokens_seen": 1568473088 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650651955867603, + "loss": 2.5811, + "theoretical_loss": 3.5023129526017973, + "tokens_seen": 1568538624 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026505516549648944, + "loss": 2.6678, + "theoretical_loss": 3.502300161228489, + "tokens_seen": 1568604160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650451354062187, + "loss": 2.3171, + "theoretical_loss": 3.5022873705392215, + "tokens_seen": 1568669696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650351053159478, + "loss": 2.3869, + "theoretical_loss": 3.5022745805339297, + "tokens_seen": 1568735232 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026502507522567704, + "loss": 2.541, + "theoretical_loss": 3.5022617912125487, + "tokens_seen": 1568800768 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650150451354062, + "loss": 2.371, + "theoretical_loss": 3.5022490025750126, + "tokens_seen": 1568866304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650050150451354, + "loss": 2.646, + "theoretical_loss": 3.5022362146212576, + "tokens_seen": 1568931840 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026499498495486464, + "loss": 2.6259, + "theoretical_loss": 3.5022234273512174, + "tokens_seen": 1568997376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026498495486459376, + "loss": 2.4813, + "theoretical_loss": 3.5022106407648277, + "tokens_seen": 1569062912 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264974924774323, + "loss": 2.5337, + "theoretical_loss": 3.5021978548620227, + "tokens_seen": 1569128448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002649648946840522, + "loss": 2.6397, + "theoretical_loss": 3.5021850696427377, + "tokens_seen": 1569193984 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026495486459378136, + "loss": 2.7135, + "theoretical_loss": 3.5021722851069077, + "tokens_seen": 1569259520 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026494483450351054, + "loss": 2.5215, + "theoretical_loss": 3.502159501254467, + "tokens_seen": 1569325056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002649348044132398, + "loss": 2.7714, + "theoretical_loss": 3.502146718085352, + "tokens_seen": 1569390592 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002649247743229689, + "loss": 2.5559, + "theoretical_loss": 3.5021339355994963, + "tokens_seen": 1569456128 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026491474423269814, + "loss": 2.5877, + "theoretical_loss": 3.5021211537968346, + "tokens_seen": 1569521664 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 867215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3438003063201904, + "objective/train/theoretical_loss": 3.502108372677303, + "objective/train/tokens_used": 1590047200, + "theoretical_loss": 3.502108372677303, + "tokens_seen": 1569587200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026490471414242727, + "loss": 2.5784, + "theoretical_loss": 3.502108372677303, + "tokens_seen": 1569587200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648946840521565, + "loss": 2.536, + "theoretical_loss": 3.5020955922408357, + "tokens_seen": 1569652736 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648846539618857, + "loss": 2.5605, + "theoretical_loss": 3.502082812487368, + "tokens_seen": 1569718272 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026487462387161486, + "loss": 2.8492, + "theoretical_loss": 3.5020700334168353, + "tokens_seen": 1569783808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026486459378134404, + "loss": 2.6621, + "theoretical_loss": 3.5020572550291718, + "tokens_seen": 1569849344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648545636910732, + "loss": 2.4295, + "theoretical_loss": 3.5020444773243122, + "tokens_seen": 1569914880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648445336008024, + "loss": 2.7394, + "theoretical_loss": 3.502031700302193, + "tokens_seen": 1569980416 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026483450351053164, + "loss": 2.6082, + "theoretical_loss": 3.5020189239627477, + "tokens_seen": 1570045952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026482447342026077, + "loss": 2.4671, + "theoretical_loss": 3.502006148305912, + "tokens_seen": 1570111488 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026481444332999, + "loss": 2.412, + "theoretical_loss": 3.5019933733316213, + "tokens_seen": 1570177024 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026480441323971913, + "loss": 2.5336, + "theoretical_loss": 3.5019805990398103, + "tokens_seen": 1570242560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026479438314944837, + "loss": 2.6074, + "theoretical_loss": 3.5019678254304143, + "tokens_seen": 1570308096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026478435305917755, + "loss": 2.5739, + "theoretical_loss": 3.5019550525033676, + "tokens_seen": 1570373632 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026477432296890673, + "loss": 2.5441, + "theoretical_loss": 3.5019422802586053, + "tokens_seen": 1570439168 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647642928786359, + "loss": 2.3627, + "theoretical_loss": 3.5019295086960636, + "tokens_seen": 1570504704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026475426278836514, + "loss": 2.5028, + "theoretical_loss": 3.501916737815677, + "tokens_seen": 1570570240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026474423269809427, + "loss": 2.5644, + "theoretical_loss": 3.5019039676173804, + "tokens_seen": 1570635776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647342026078235, + "loss": 2.6869, + "theoretical_loss": 3.501891198101109, + "tokens_seen": 1570701312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026472417251755263, + "loss": 2.6599, + "theoretical_loss": 3.5018784292667977, + "tokens_seen": 1570766848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026471414242728187, + "loss": 2.4728, + "theoretical_loss": 3.5018656611143824, + "tokens_seen": 1570832384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026470411233701105, + "loss": 2.6884, + "theoretical_loss": 3.5018528936437976, + "tokens_seen": 1570897920 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026469408224674023, + "loss": 2.5154, + "theoretical_loss": 3.5018401268549786, + "tokens_seen": 1570963456 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646840521564694, + "loss": 2.728, + "theoretical_loss": 3.5018273607478605, + "tokens_seen": 1571028992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646740220661986, + "loss": 2.5455, + "theoretical_loss": 3.5018145953223785, + "tokens_seen": 1571094528 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646639919759278, + "loss": 2.7501, + "theoretical_loss": 3.5018018305784677, + "tokens_seen": 1571160064 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 868323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.522857189178467, + "objective/train/theoretical_loss": 3.5017890665160634, + "objective/train/tokens_used": 1591685600, + "theoretical_loss": 3.5017890665160634, + "tokens_seen": 1571225600 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264653961885657, + "loss": 2.6023, + "theoretical_loss": 3.5017890665160634, + "tokens_seen": 1571225600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026464393179538614, + "loss": 2.6551, + "theoretical_loss": 3.5017763031351006, + "tokens_seen": 1571291136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026463390170511537, + "loss": 2.7338, + "theoretical_loss": 3.5017635404355154, + "tokens_seen": 1571356672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646238716148445, + "loss": 2.4192, + "theoretical_loss": 3.5017507784172412, + "tokens_seen": 1571422208 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026461384152457373, + "loss": 2.6184, + "theoretical_loss": 3.501738017080215, + "tokens_seen": 1571487744 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646038114343029, + "loss": 2.5038, + "theoretical_loss": 3.501725256424371, + "tokens_seen": 1571553280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645937813440321, + "loss": 2.4952, + "theoretical_loss": 3.5017124964496453, + "tokens_seen": 1571618816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645837512537613, + "loss": 2.6781, + "theoretical_loss": 3.501699737155972, + "tokens_seen": 1571684352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645737211634905, + "loss": 2.4646, + "theoretical_loss": 3.501686978543287, + "tokens_seen": 1571749888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026456369107321964, + "loss": 2.6156, + "theoretical_loss": 3.501674220611526, + "tokens_seen": 1571815424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645536609829489, + "loss": 2.455, + "theoretical_loss": 3.501661463360623, + "tokens_seen": 1571880960 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264543630892678, + "loss": 2.7022, + "theoretical_loss": 3.5016487067905144, + "tokens_seen": 1571946496 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026453360080240724, + "loss": 2.5932, + "theoretical_loss": 3.501635950901135, + "tokens_seen": 1572012032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645235707121364, + "loss": 2.3791, + "theoretical_loss": 3.501623195692421, + "tokens_seen": 1572077568 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645135406218656, + "loss": 2.6157, + "theoretical_loss": 3.5016104411643063, + "tokens_seen": 1572143104 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645035105315948, + "loss": 2.5162, + "theoretical_loss": 3.501597687316727, + "tokens_seen": 1572208640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026449348044132396, + "loss": 2.5154, + "theoretical_loss": 3.5015849341496184, + "tokens_seen": 1572274176 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026448345035105314, + "loss": 2.6475, + "theoretical_loss": 3.5015721816629153, + "tokens_seen": 1572339712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644734202607824, + "loss": 2.5634, + "theoretical_loss": 3.5015594298565538, + "tokens_seen": 1572405248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644633901705115, + "loss": 2.3788, + "theoretical_loss": 3.5015466787304685, + "tokens_seen": 1572470784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026445336008024074, + "loss": 2.6087, + "theoretical_loss": 3.5015339282845956, + "tokens_seen": 1572536320 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026444332998996987, + "loss": 2.6118, + "theoretical_loss": 3.5015211785188702, + "tokens_seen": 1572601856 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644332998996991, + "loss": 2.7455, + "theoretical_loss": 3.501508429433227, + "tokens_seen": 1572667392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644232698094283, + "loss": 2.5472, + "theoretical_loss": 3.5014956810276026, + "tokens_seen": 1572732928 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026441323971915747, + "loss": 2.5672, + "theoretical_loss": 3.501482933301931, + "tokens_seen": 1572798464 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 869061, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5812792778015137, + "objective/train/theoretical_loss": 3.5014701862561486, + "objective/train/tokens_used": 1593324000, + "theoretical_loss": 3.5014701862561486, + "tokens_seen": 1572864000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026440320962888665, + "loss": 2.5332, + "theoretical_loss": 3.5014701862561486, + "tokens_seen": 1572864000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643931795386159, + "loss": 2.4261, + "theoretical_loss": 3.5014574398901903, + "tokens_seen": 1572929536 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264383149448345, + "loss": 2.5914, + "theoretical_loss": 3.5014446942039923, + "tokens_seen": 1572995072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026437311935807424, + "loss": 2.5054, + "theoretical_loss": 3.501431949197489, + "tokens_seen": 1573060608 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026436308926780337, + "loss": 2.2896, + "theoretical_loss": 3.501419204870616, + "tokens_seen": 1573126144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643530591775326, + "loss": 2.5589, + "theoretical_loss": 3.5014064612233096, + "tokens_seen": 1573191680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643430290872618, + "loss": 2.4529, + "theoretical_loss": 3.5013937182555046, + "tokens_seen": 1573257216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026433299899699097, + "loss": 2.713, + "theoretical_loss": 3.5013809759671366, + "tokens_seen": 1573322752 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026432296890672015, + "loss": 2.6435, + "theoretical_loss": 3.5013682343581407, + "tokens_seen": 1573388288 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026431293881644933, + "loss": 2.4365, + "theoretical_loss": 3.501355493428453, + "tokens_seen": 1573453824 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643029087261785, + "loss": 2.5929, + "theoretical_loss": 3.501342753178009, + "tokens_seen": 1573519360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026429287863590775, + "loss": 2.6807, + "theoretical_loss": 3.5013300136067436, + "tokens_seen": 1573584896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642828485456369, + "loss": 2.5183, + "theoretical_loss": 3.5013172747145926, + "tokens_seen": 1573650432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642728184553661, + "loss": 2.5825, + "theoretical_loss": 3.5013045365014914, + "tokens_seen": 1573715968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026426278836509534, + "loss": 2.493, + "theoretical_loss": 3.501291798967376, + "tokens_seen": 1573781504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026425275827482447, + "loss": 2.2586, + "theoretical_loss": 3.5012790621121814, + "tokens_seen": 1573847040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642427281845537, + "loss": 2.4789, + "theoretical_loss": 3.501266325935844, + "tokens_seen": 1573912576 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026423269809428283, + "loss": 2.5391, + "theoretical_loss": 3.5012535904382975, + "tokens_seen": 1573978112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026422266800401207, + "loss": 2.5963, + "theoretical_loss": 3.5012408556194794, + "tokens_seen": 1574043648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026421263791374125, + "loss": 2.5903, + "theoretical_loss": 3.5012281214793246, + "tokens_seen": 1574109184 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026420260782347043, + "loss": 2.5422, + "theoretical_loss": 3.5012153880177683, + "tokens_seen": 1574174720 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641925777331996, + "loss": 2.4667, + "theoretical_loss": 3.5012026552347466, + "tokens_seen": 1574240256 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641825476429288, + "loss": 2.3859, + "theoretical_loss": 3.501189923130195, + "tokens_seen": 1574305792 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264172517552658, + "loss": 2.4377, + "theoretical_loss": 3.5011771917040493, + "tokens_seen": 1574371328 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641624874623872, + "loss": 2.8072, + "theoretical_loss": 3.501164460956244, + "tokens_seen": 1574436864 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 869692, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4734344482421875, + "objective/train/theoretical_loss": 3.501151730886716, + "objective/train/tokens_used": 1594962400, + "theoretical_loss": 3.501151730886716, + "tokens_seen": 1574502400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026415245737211634, + "loss": 2.6231, + "theoretical_loss": 3.501151730886716, + "tokens_seen": 1574502400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026414242728184557, + "loss": 2.5573, + "theoretical_loss": 3.5011390014954005, + "tokens_seen": 1574567936 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641323971915747, + "loss": 2.6861, + "theoretical_loss": 3.5011262727822334, + "tokens_seen": 1574633472 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026412236710130393, + "loss": 2.4734, + "theoretical_loss": 3.5011135447471498, + "tokens_seen": 1574699008 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641123370110331, + "loss": 2.515, + "theoretical_loss": 3.5011008173900855, + "tokens_seen": 1574764544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641023069207623, + "loss": 2.5897, + "theoretical_loss": 3.5010880907109767, + "tokens_seen": 1574830080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640922768304915, + "loss": 2.3304, + "theoretical_loss": 3.501075364709758, + "tokens_seen": 1574895616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640822467402207, + "loss": 2.5674, + "theoretical_loss": 3.501062639386366, + "tokens_seen": 1574961152 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026407221664994984, + "loss": 2.8064, + "theoretical_loss": 3.5010499147407366, + "tokens_seen": 1575026688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640621865596791, + "loss": 2.657, + "theoretical_loss": 3.501037190772805, + "tokens_seen": 1575092224 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640521564694082, + "loss": 2.8227, + "theoretical_loss": 3.501024467482507, + "tokens_seen": 1575157760 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026404212637913744, + "loss": 2.5477, + "theoretical_loss": 3.5010117448697775, + "tokens_seen": 1575223296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640320962888666, + "loss": 2.5243, + "theoretical_loss": 3.500999022934554, + "tokens_seen": 1575288832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640220661985958, + "loss": 2.6759, + "theoretical_loss": 3.500986301676771, + "tokens_seen": 1575354368 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264012036108325, + "loss": 2.6374, + "theoretical_loss": 3.500973581096364, + "tokens_seen": 1575419904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026400200601805416, + "loss": 2.771, + "theoretical_loss": 3.5009608611932697, + "tokens_seen": 1575485440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026399197592778334, + "loss": 2.5815, + "theoretical_loss": 3.500948141967423, + "tokens_seen": 1575550976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639819458375126, + "loss": 2.4938, + "theoretical_loss": 3.5009354234187606, + "tokens_seen": 1575616512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639719157472417, + "loss": 2.854, + "theoretical_loss": 3.5009227055472176, + "tokens_seen": 1575682048 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026396188565697094, + "loss": 2.3283, + "theoretical_loss": 3.50090998835273, + "tokens_seen": 1575747584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026395185556670007, + "loss": 2.5448, + "theoretical_loss": 3.5008972718352336, + "tokens_seen": 1575813120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639418254764293, + "loss": 2.8598, + "theoretical_loss": 3.500884555994664, + "tokens_seen": 1575878656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639317953861585, + "loss": 2.4873, + "theoretical_loss": 3.500871840830958, + "tokens_seen": 1575944192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026392176529588767, + "loss": 2.3254, + "theoretical_loss": 3.500859126344049, + "tokens_seen": 1576009728 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026391173520561685, + "loss": 2.6268, + "theoretical_loss": 3.5008464125338756, + "tokens_seen": 1576075264 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 870886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7491490840911865, + "objective/train/theoretical_loss": 3.5008336994003724, + "objective/train/tokens_used": 1596600800, + "theoretical_loss": 3.5008336994003724, + "tokens_seen": 1576140800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639017051153461, + "loss": 2.7081, + "theoretical_loss": 3.5008336994003724, + "tokens_seen": 1576140800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638916750250752, + "loss": 2.4539, + "theoretical_loss": 3.5008209869434754, + "tokens_seen": 1576206336 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026388164493480444, + "loss": 2.5672, + "theoretical_loss": 3.50080827516312, + "tokens_seen": 1576271872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026387161484453357, + "loss": 2.4068, + "theoretical_loss": 3.500795564059243, + "tokens_seen": 1576337408 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638615847542628, + "loss": 2.5358, + "theoretical_loss": 3.5007828536317795, + "tokens_seen": 1576402944 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263851554663992, + "loss": 2.4222, + "theoretical_loss": 3.5007701438806658, + "tokens_seen": 1576468480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026384152457372117, + "loss": 2.513, + "theoretical_loss": 3.5007574348058372, + "tokens_seen": 1576534016 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026383149448345035, + "loss": 2.6312, + "theoretical_loss": 3.50074472640723, + "tokens_seen": 1576599552 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026382146439317953, + "loss": 2.492, + "theoretical_loss": 3.5007320186847806, + "tokens_seen": 1576665088 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638114343029087, + "loss": 2.3217, + "theoretical_loss": 3.5007193116384245, + "tokens_seen": 1576730624 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026380140421263795, + "loss": 2.574, + "theoretical_loss": 3.500706605268097, + "tokens_seen": 1576796160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637913741223671, + "loss": 2.72, + "theoretical_loss": 3.5006938995737356, + "tokens_seen": 1576861696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637813440320963, + "loss": 2.4526, + "theoretical_loss": 3.5006811945552743, + "tokens_seen": 1576927232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637713139418255, + "loss": 2.38, + "theoretical_loss": 3.500668490212651, + "tokens_seen": 1576992768 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026376128385155467, + "loss": 2.3041, + "theoretical_loss": 3.5006557865458, + "tokens_seen": 1577058304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026375125376128385, + "loss": 2.5886, + "theoretical_loss": 3.5006430835546585, + "tokens_seen": 1577123840 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026374122367101303, + "loss": 2.8158, + "theoretical_loss": 3.5006303812391613, + "tokens_seen": 1577189376 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637311935807422, + "loss": 2.4255, + "theoretical_loss": 3.5006176795992454, + "tokens_seen": 1577254912 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026372116349047145, + "loss": 2.5985, + "theoretical_loss": 3.500604978634847, + "tokens_seen": 1577320448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637111334002006, + "loss": 2.5509, + "theoretical_loss": 3.500592278345901, + "tokens_seen": 1577385984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637011033099298, + "loss": 2.5023, + "theoretical_loss": 3.500579578732344, + "tokens_seen": 1577451520 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026369107321965894, + "loss": 2.6507, + "theoretical_loss": 3.5005668797941123, + "tokens_seen": 1577517056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636810431293882, + "loss": 2.5937, + "theoretical_loss": 3.5005541815311414, + "tokens_seen": 1577582592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026367101303911736, + "loss": 2.7392, + "theoretical_loss": 3.5005414839433673, + "tokens_seen": 1577648128 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026366098294884654, + "loss": 2.6395, + "theoretical_loss": 3.500528787030727, + "tokens_seen": 1577713664 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 871486, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.030456781387329, + "objective/train/theoretical_loss": 3.5005160907931554, + "objective/train/tokens_used": 1598239200, + "theoretical_loss": 3.5005160907931554, + "tokens_seen": 1577779200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636509528585757, + "loss": 3.0241, + "theoretical_loss": 3.5005160907931554, + "tokens_seen": 1577779200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636409227683049, + "loss": 2.5359, + "theoretical_loss": 3.5005033952305893, + "tokens_seen": 1577844736 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636308926780341, + "loss": 2.6299, + "theoretical_loss": 3.5004907003429646, + "tokens_seen": 1577910272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636208625877633, + "loss": 2.6755, + "theoretical_loss": 3.5004780061302174, + "tokens_seen": 1577975808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026361083249749244, + "loss": 2.5483, + "theoretical_loss": 3.5004653125922838, + "tokens_seen": 1578041344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636008024072217, + "loss": 2.6397, + "theoretical_loss": 3.5004526197290993, + "tokens_seen": 1578106880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026359077231695086, + "loss": 2.5487, + "theoretical_loss": 3.500439927540601, + "tokens_seen": 1578172416 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026358074222668004, + "loss": 2.5478, + "theoretical_loss": 3.5004272360267246, + "tokens_seen": 1578237952 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635707121364092, + "loss": 2.3817, + "theoretical_loss": 3.500414545187406, + "tokens_seen": 1578303488 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635606820461384, + "loss": 2.562, + "theoretical_loss": 3.5004018550225817, + "tokens_seen": 1578369024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635506519558676, + "loss": 2.3398, + "theoretical_loss": 3.5003891655321877, + "tokens_seen": 1578434560 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635406218655968, + "loss": 2.5235, + "theoretical_loss": 3.50037647671616, + "tokens_seen": 1578500096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026353059177532595, + "loss": 2.6254, + "theoretical_loss": 3.500363788574435, + "tokens_seen": 1578565632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635205616850552, + "loss": 2.5682, + "theoretical_loss": 3.5003511011069492, + "tokens_seen": 1578631168 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026351053159478436, + "loss": 2.624, + "theoretical_loss": 3.500338414313638, + "tokens_seen": 1578696704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026350050150451354, + "loss": 2.5264, + "theoretical_loss": 3.500325728194438, + "tokens_seen": 1578762240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634904714142428, + "loss": 2.5816, + "theoretical_loss": 3.5003130427492852, + "tokens_seen": 1578827776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634804413239719, + "loss": 2.5874, + "theoretical_loss": 3.5003003579781162, + "tokens_seen": 1578893312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026347041123370114, + "loss": 2.5642, + "theoretical_loss": 3.500287673880867, + "tokens_seen": 1578958848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026346038114343027, + "loss": 2.4428, + "theoretical_loss": 3.500274990457474, + "tokens_seen": 1579024384 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634503510531595, + "loss": 2.6323, + "theoretical_loss": 3.5002623077078727, + "tokens_seen": 1579089920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634403209628887, + "loss": 2.7493, + "theoretical_loss": 3.5002496256320006, + "tokens_seen": 1579155456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026343029087261787, + "loss": 2.3776, + "theoretical_loss": 3.5002369442297927, + "tokens_seen": 1579220992 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026342026078234705, + "loss": 2.4109, + "theoretical_loss": 3.5002242635011864, + "tokens_seen": 1579286528 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634102306920763, + "loss": 2.5289, + "theoretical_loss": 3.5002115834461165, + "tokens_seen": 1579352064 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 872887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.328456401824951, + "objective/train/theoretical_loss": 3.500198904064521, + "objective/train/tokens_used": 1599877600, + "theoretical_loss": 3.500198904064521, + "tokens_seen": 1579417600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634002006018054, + "loss": 2.5613, + "theoretical_loss": 3.500198904064521, + "tokens_seen": 1579417600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026339017051153464, + "loss": 2.4966, + "theoretical_loss": 3.5001862253563347, + "tokens_seen": 1579483136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026338014042126377, + "loss": 2.5745, + "theoretical_loss": 3.500173547321495, + "tokens_seen": 1579548672 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263370110330993, + "loss": 2.695, + "theoretical_loss": 3.500160869959938, + "tokens_seen": 1579614208 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002633600802407222, + "loss": 2.5789, + "theoretical_loss": 3.500148193271599, + "tokens_seen": 1579679744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026335005015045137, + "loss": 2.6317, + "theoretical_loss": 3.5001355172564157, + "tokens_seen": 1579745280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026334002006018055, + "loss": 2.4367, + "theoretical_loss": 3.5001228419143233, + "tokens_seen": 1579810816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026332998996990973, + "loss": 2.4399, + "theoretical_loss": 3.5001101672452593, + "tokens_seen": 1579876352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002633199598796389, + "loss": 2.3521, + "theoretical_loss": 3.500097493249159, + "tokens_seen": 1579941888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026330992978936815, + "loss": 2.3943, + "theoretical_loss": 3.5000848199259593, + "tokens_seen": 1580007424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632998996990973, + "loss": 2.5588, + "theoretical_loss": 3.5000721472755965, + "tokens_seen": 1580072960 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632898696088265, + "loss": 2.6983, + "theoretical_loss": 3.5000594752980065, + "tokens_seen": 1580138496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632798395185557, + "loss": 2.5467, + "theoretical_loss": 3.5000468039931265, + "tokens_seen": 1580204032 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026326980942828487, + "loss": 2.529, + "theoretical_loss": 3.5000341333608924, + "tokens_seen": 1580269568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026325977933801405, + "loss": 2.5671, + "theoretical_loss": 3.5000214634012403, + "tokens_seen": 1580335104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026324974924774323, + "loss": 2.5333, + "theoretical_loss": 3.500008794114107, + "tokens_seen": 1580400640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632397191574724, + "loss": 2.5233, + "theoretical_loss": 3.49999612549943, + "tokens_seen": 1580466176 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026322968906720165, + "loss": 2.6768, + "theoretical_loss": 3.4999834575571436, + "tokens_seen": 1580531712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632196589769308, + "loss": 2.5567, + "theoretical_loss": 3.4999707902871853, + "tokens_seen": 1580597248 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026320962888666, + "loss": 2.6232, + "theoretical_loss": 3.499958123689492, + "tokens_seen": 1580662784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026319959879638914, + "loss": 2.447, + "theoretical_loss": 3.499945457763999, + "tokens_seen": 1580728320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631895687061184, + "loss": 2.5432, + "theoretical_loss": 3.499932792510644, + "tokens_seen": 1580793856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026317953861584756, + "loss": 2.6017, + "theoretical_loss": 3.4999201279293626, + "tokens_seen": 1580859392 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026316950852557674, + "loss": 2.6208, + "theoretical_loss": 3.4999074640200916, + "tokens_seen": 1580924928 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631594784353059, + "loss": 2.8345, + "theoretical_loss": 3.4998948007827675, + "tokens_seen": 1580990464 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 874187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.422612428665161, + "objective/train/theoretical_loss": 3.4998821382173264, + "objective/train/tokens_used": 1601516000, + "theoretical_loss": 3.4998821382173264, + "tokens_seen": 1581056000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631494483450351, + "loss": 2.5285, + "theoretical_loss": 3.4998821382173264, + "tokens_seen": 1581056000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631394182547643, + "loss": 2.6874, + "theoretical_loss": 3.4998694763237053, + "tokens_seen": 1581121536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631293881644935, + "loss": 2.4081, + "theoretical_loss": 3.4998568151018405, + "tokens_seen": 1581187072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026311935807422264, + "loss": 2.5127, + "theoretical_loss": 3.499844154551669, + "tokens_seen": 1581252608 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631093279839519, + "loss": 2.4452, + "theoretical_loss": 3.499831494673127, + "tokens_seen": 1581318144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026309929789368106, + "loss": 2.4203, + "theoretical_loss": 3.4998188354661504, + "tokens_seen": 1581383680 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026308926780341024, + "loss": 2.3916, + "theoretical_loss": 3.4998061769306767, + "tokens_seen": 1581449216 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630792377131394, + "loss": 2.4841, + "theoretical_loss": 3.4997935190666416, + "tokens_seen": 1581514752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630692076228686, + "loss": 2.4694, + "theoretical_loss": 3.4997808618739823, + "tokens_seen": 1581580288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630591775325978, + "loss": 2.4382, + "theoretical_loss": 3.4997682053526358, + "tokens_seen": 1581645824 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263049147442327, + "loss": 2.5936, + "theoretical_loss": 3.499755549502537, + "tokens_seen": 1581711360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026303911735205615, + "loss": 2.7564, + "theoretical_loss": 3.4997428943236244, + "tokens_seen": 1581776896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630290872617854, + "loss": 2.6199, + "theoretical_loss": 3.4997302398158334, + "tokens_seen": 1581842432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630190571715145, + "loss": 2.8209, + "theoretical_loss": 3.499717585979101, + "tokens_seen": 1581907968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026300902708124374, + "loss": 2.5797, + "theoretical_loss": 3.499704932813364, + "tokens_seen": 1581973504 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629989969909729, + "loss": 2.4459, + "theoretical_loss": 3.499692280318558, + "tokens_seen": 1582039040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629889669007021, + "loss": 2.7433, + "theoretical_loss": 3.499679628494621, + "tokens_seen": 1582104576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629789368104313, + "loss": 2.3748, + "theoretical_loss": 3.4996669773414895, + "tokens_seen": 1582170112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026296890672016047, + "loss": 2.4761, + "theoretical_loss": 3.4996543268590994, + "tokens_seen": 1582235648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026295887662988965, + "loss": 2.5291, + "theoretical_loss": 3.4996416770473875, + "tokens_seen": 1582301184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629488465396189, + "loss": 2.2966, + "theoretical_loss": 3.4996290279062907, + "tokens_seen": 1582366720 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262938816449348, + "loss": 2.65, + "theoretical_loss": 3.4996163794357456, + "tokens_seen": 1582432256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026292878635907725, + "loss": 2.5125, + "theoretical_loss": 3.4996037316356894, + "tokens_seen": 1582497792 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026291875626880643, + "loss": 2.7155, + "theoretical_loss": 3.499591084506058, + "tokens_seen": 1582563328 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629087261785356, + "loss": 2.4625, + "theoretical_loss": 3.499578438046788, + "tokens_seen": 1582628864 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 874795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7404651641845703, + "objective/train/theoretical_loss": 3.499565792257817, + "objective/train/tokens_used": 1603154400, + "theoretical_loss": 3.499565792257817, + "tokens_seen": 1582694400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628986960882648, + "loss": 2.7593, + "theoretical_loss": 3.499565792257817, + "tokens_seen": 1582694400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026288866599799397, + "loss": 2.5075, + "theoretical_loss": 3.499553147139081, + "tokens_seen": 1582759936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026287863590772315, + "loss": 2.8189, + "theoretical_loss": 3.499540502690517, + "tokens_seen": 1582825472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628686058174524, + "loss": 2.4139, + "theoretical_loss": 3.4995278589120615, + "tokens_seen": 1582891008 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628585757271815, + "loss": 2.5647, + "theoretical_loss": 3.499515215803652, + "tokens_seen": 1582956544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026284854563691075, + "loss": 2.5506, + "theoretical_loss": 3.4995025733652243, + "tokens_seen": 1583022080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628385155466399, + "loss": 2.525, + "theoretical_loss": 3.4994899315967154, + "tokens_seen": 1583087616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628284854563691, + "loss": 2.4954, + "theoretical_loss": 3.4994772904980627, + "tokens_seen": 1583153152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628184553660983, + "loss": 2.4888, + "theoretical_loss": 3.4994646500692017, + "tokens_seen": 1583218688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628084252758275, + "loss": 2.4868, + "theoretical_loss": 3.499452010310071, + "tokens_seen": 1583284224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026279839518555666, + "loss": 2.2767, + "theoretical_loss": 3.499439371220606, + "tokens_seen": 1583349760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627883650952859, + "loss": 2.7694, + "theoretical_loss": 3.4994267328007433, + "tokens_seen": 1583415296 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262778335005015, + "loss": 2.5017, + "theoretical_loss": 3.499414095050421, + "tokens_seen": 1583480832 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026276830491474425, + "loss": 2.6436, + "theoretical_loss": 3.499401457969575, + "tokens_seen": 1583546368 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026275827482447343, + "loss": 2.5037, + "theoretical_loss": 3.4993888215581426, + "tokens_seen": 1583611904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627482447342026, + "loss": 2.8348, + "theoretical_loss": 3.49937618581606, + "tokens_seen": 1583677440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026273821464393185, + "loss": 2.6411, + "theoretical_loss": 3.499363550743265, + "tokens_seen": 1583742976 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262728184553661, + "loss": 2.5223, + "theoretical_loss": 3.4993509163396936, + "tokens_seen": 1583808512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627181544633902, + "loss": 2.4816, + "theoretical_loss": 3.499338282605283, + "tokens_seen": 1583874048 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026270812437311934, + "loss": 2.5636, + "theoretical_loss": 3.49932564953997, + "tokens_seen": 1583939584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626980942828486, + "loss": 2.6754, + "theoretical_loss": 3.4993130171436917, + "tokens_seen": 1584005120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026268806419257776, + "loss": 2.8138, + "theoretical_loss": 3.4993003854163844, + "tokens_seen": 1584070656 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026267803410230694, + "loss": 2.6237, + "theoretical_loss": 3.4992877543579857, + "tokens_seen": 1584136192 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626680040120361, + "loss": 2.4506, + "theoretical_loss": 3.4992751239684323, + "tokens_seen": 1584201728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626579739217653, + "loss": 2.6602, + "theoretical_loss": 3.499262494247661, + "tokens_seen": 1584267264 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 875946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.513859510421753, + "objective/train/theoretical_loss": 3.4992498651956088, + "objective/train/tokens_used": 1604792800, + "theoretical_loss": 3.4992498651956088, + "tokens_seen": 1584332800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626479438314945, + "loss": 2.3576, + "theoretical_loss": 3.4992498651956088, + "tokens_seen": 1584332800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626379137412237, + "loss": 2.5001, + "theoretical_loss": 3.4992372368122124, + "tokens_seen": 1584398336 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026262788365095284, + "loss": 2.5837, + "theoretical_loss": 3.499224609097409, + "tokens_seen": 1584463872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626178535606821, + "loss": 2.7267, + "theoretical_loss": 3.4992119820511354, + "tokens_seen": 1584529408 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026260782347041126, + "loss": 2.3748, + "theoretical_loss": 3.499199355673329, + "tokens_seen": 1584594944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026259779338014044, + "loss": 2.6236, + "theoretical_loss": 3.4991867299639257, + "tokens_seen": 1584660480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625877632898696, + "loss": 2.7491, + "theoretical_loss": 3.499174104922864, + "tokens_seen": 1584726016 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625777331995988, + "loss": 2.3001, + "theoretical_loss": 3.4991614805500797, + "tokens_seen": 1584791552 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262567703109328, + "loss": 2.7681, + "theoretical_loss": 3.49914885684551, + "tokens_seen": 1584857088 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625576730190572, + "loss": 2.644, + "theoretical_loss": 3.499136233809092, + "tokens_seen": 1584922624 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026254764292878635, + "loss": 2.2981, + "theoretical_loss": 3.4991236114407633, + "tokens_seen": 1584988160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625376128385156, + "loss": 2.6859, + "theoretical_loss": 3.49911098974046, + "tokens_seen": 1585053696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625275827482447, + "loss": 2.6707, + "theoretical_loss": 3.49909836870812, + "tokens_seen": 1585119232 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026251755265797394, + "loss": 2.5614, + "theoretical_loss": 3.4990857483436795, + "tokens_seen": 1585184768 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625075225677031, + "loss": 2.3751, + "theoretical_loss": 3.4990731286470758, + "tokens_seen": 1585250304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624974924774323, + "loss": 2.752, + "theoretical_loss": 3.4990605096182463, + "tokens_seen": 1585315840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624874623871615, + "loss": 2.6411, + "theoretical_loss": 3.4990478912571277, + "tokens_seen": 1585381376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026247743229689067, + "loss": 2.6529, + "theoretical_loss": 3.499035273563657, + "tokens_seen": 1585446912 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026246740220661985, + "loss": 2.6479, + "theoretical_loss": 3.499022656537772, + "tokens_seen": 1585512448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624573721163491, + "loss": 2.6785, + "theoretical_loss": 3.4990100401794084, + "tokens_seen": 1585577984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624473420260782, + "loss": 2.5063, + "theoretical_loss": 3.498997424488505, + "tokens_seen": 1585643520 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026243731193580745, + "loss": 2.7766, + "theoretical_loss": 3.498984809464998, + "tokens_seen": 1585709056 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026242728184553663, + "loss": 2.7247, + "theoretical_loss": 3.498972195108824, + "tokens_seen": 1585774592 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624172517552658, + "loss": 2.5592, + "theoretical_loss": 3.4989595814199212, + "tokens_seen": 1585840128 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262407221664995, + "loss": 2.7065, + "theoretical_loss": 3.4989469683982257, + "tokens_seen": 1585905664 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 876427, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.855236053466797, + "objective/train/theoretical_loss": 3.4989343560436756, + "objective/train/tokens_used": 1606431200, + "theoretical_loss": 3.4989343560436756, + "tokens_seen": 1585971200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026239719157472417, + "loss": 2.5245, + "theoretical_loss": 3.4989343560436756, + "tokens_seen": 1585971200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026238716148445335, + "loss": 2.444, + "theoretical_loss": 3.4989217443562075, + "tokens_seen": 1586036736 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623771313941826, + "loss": 2.449, + "theoretical_loss": 3.4989091333357587, + "tokens_seen": 1586102272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623671013039117, + "loss": 2.5151, + "theoretical_loss": 3.498896522982266, + "tokens_seen": 1586167808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026235707121364095, + "loss": 2.4415, + "theoretical_loss": 3.4988839132956673, + "tokens_seen": 1586233344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623470411233701, + "loss": 2.6342, + "theoretical_loss": 3.498871304275899, + "tokens_seen": 1586298880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623370110330993, + "loss": 2.492, + "theoretical_loss": 3.498858695922899, + "tokens_seen": 1586364416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623269809428285, + "loss": 2.7537, + "theoretical_loss": 3.498846088236604, + "tokens_seen": 1586429952 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623169508525577, + "loss": 2.4092, + "theoretical_loss": 3.4988334812169515, + "tokens_seen": 1586495488 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026230692076228686, + "loss": 2.4392, + "theoretical_loss": 3.4988208748638785, + "tokens_seen": 1586561024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622968906720161, + "loss": 2.4274, + "theoretical_loss": 3.4988082691773226, + "tokens_seen": 1586626560 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622868605817452, + "loss": 2.6292, + "theoretical_loss": 3.49879566415722, + "tokens_seen": 1586692096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026227683049147445, + "loss": 2.6117, + "theoretical_loss": 3.4987830598035092, + "tokens_seen": 1586757632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622668004012036, + "loss": 2.8439, + "theoretical_loss": 3.498770456116127, + "tokens_seen": 1586823168 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622567703109328, + "loss": 2.6063, + "theoretical_loss": 3.49875785309501, + "tokens_seen": 1586888704 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262246740220662, + "loss": 2.6568, + "theoretical_loss": 3.4987452507400967, + "tokens_seen": 1586954240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622367101303912, + "loss": 2.6322, + "theoretical_loss": 3.4987326490513238, + "tokens_seen": 1587019776 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026222668004012036, + "loss": 2.299, + "theoretical_loss": 3.498720048028628, + "tokens_seen": 1587085312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026221664994984954, + "loss": 2.609, + "theoretical_loss": 3.498707447671947, + "tokens_seen": 1587150848 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622066198595787, + "loss": 2.7251, + "theoretical_loss": 3.498694847981219, + "tokens_seen": 1587216384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026219658976930796, + "loss": 2.4335, + "theoretical_loss": 3.49868224895638, + "tokens_seen": 1587281920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621865596790371, + "loss": 2.5573, + "theoretical_loss": 3.4986696505973676, + "tokens_seen": 1587347456 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621765295887663, + "loss": 2.6674, + "theoretical_loss": 3.49865705290412, + "tokens_seen": 1587412992 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026216649949849545, + "loss": 2.6773, + "theoretical_loss": 3.4986444558765735, + "tokens_seen": 1587478528 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621564694082247, + "loss": 2.4559, + "theoretical_loss": 3.4986318595146653, + "tokens_seen": 1587544064 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 877583, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5934700965881348, + "objective/train/theoretical_loss": 3.498619263818334, + "objective/train/tokens_used": 1608069600, + "theoretical_loss": 3.498619263818334, + "tokens_seen": 1587609600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026214643931795386, + "loss": 2.3889, + "theoretical_loss": 3.498619263818334, + "tokens_seen": 1587609600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026213640922768304, + "loss": 2.5724, + "theoretical_loss": 3.4986066687875157, + "tokens_seen": 1587675136 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621263791374122, + "loss": 2.4605, + "theoretical_loss": 3.498594074422149, + "tokens_seen": 1587740672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026211634904714146, + "loss": 2.5194, + "theoretical_loss": 3.4985814807221702, + "tokens_seen": 1587806208 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621063189568706, + "loss": 2.5424, + "theoretical_loss": 3.498568887687517, + "tokens_seen": 1587871744 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620962888665998, + "loss": 2.4043, + "theoretical_loss": 3.498556295318127, + "tokens_seen": 1587937280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026208625877632895, + "loss": 2.6208, + "theoretical_loss": 3.4985437036139375, + "tokens_seen": 1588002816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620762286860582, + "loss": 2.6661, + "theoretical_loss": 3.4985311125748852, + "tokens_seen": 1588068352 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026206619859578737, + "loss": 2.3886, + "theoretical_loss": 3.498518522200909, + "tokens_seen": 1588133888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026205616850551655, + "loss": 2.6985, + "theoretical_loss": 3.498505932491945, + "tokens_seen": 1588199424 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026204613841524573, + "loss": 2.5593, + "theoretical_loss": 3.4984933434479313, + "tokens_seen": 1588264960 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620361083249749, + "loss": 2.819, + "theoretical_loss": 3.4984807550688055, + "tokens_seen": 1588330496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620260782347041, + "loss": 2.5413, + "theoretical_loss": 3.4984681673545044, + "tokens_seen": 1588396032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620160481444333, + "loss": 2.6242, + "theoretical_loss": 3.4984555803049657, + "tokens_seen": 1588461568 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620060180541625, + "loss": 2.2904, + "theoretical_loss": 3.4984429939201274, + "tokens_seen": 1588527104 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619959879638917, + "loss": 2.5513, + "theoretical_loss": 3.4984304081999262, + "tokens_seen": 1588592640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026198595787362087, + "loss": 2.7006, + "theoretical_loss": 3.4984178231443, + "tokens_seen": 1588658176 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026197592778335005, + "loss": 2.8216, + "theoretical_loss": 3.498405238753187, + "tokens_seen": 1588723712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619658976930793, + "loss": 2.2396, + "theoretical_loss": 3.498392655026523, + "tokens_seen": 1588789248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619558676028084, + "loss": 2.4445, + "theoretical_loss": 3.4983800719642466, + "tokens_seen": 1588854784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026194583751253765, + "loss": 2.5329, + "theoretical_loss": 3.4983674895662955, + "tokens_seen": 1588920320 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026193580742226683, + "loss": 2.5839, + "theoretical_loss": 3.4983549078326064, + "tokens_seen": 1588985856 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261925777331996, + "loss": 2.563, + "theoretical_loss": 3.498342326763118, + "tokens_seen": 1589051392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619157472417252, + "loss": 2.5279, + "theoretical_loss": 3.4983297463577667, + "tokens_seen": 1589116928 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026190571715145437, + "loss": 2.613, + "theoretical_loss": 3.4983171666164905, + "tokens_seen": 1589182464 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 878291, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9362738132476807, + "objective/train/theoretical_loss": 3.498304587539227, + "objective/train/tokens_used": 1609708000, + "theoretical_loss": 3.498304587539227, + "tokens_seen": 1589248000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026189568706118355, + "loss": 2.7078, + "theoretical_loss": 3.498304587539227, + "tokens_seen": 1589248000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618856569709128, + "loss": 2.4795, + "theoretical_loss": 3.498292009125914, + "tokens_seen": 1589313536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618756268806419, + "loss": 2.5528, + "theoretical_loss": 3.498279431376489, + "tokens_seen": 1589379072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026186559679037115, + "loss": 2.3938, + "theoretical_loss": 3.498266854290889, + "tokens_seen": 1589444608 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618555667001003, + "loss": 2.7332, + "theoretical_loss": 3.4982542778690524, + "tokens_seen": 1589510144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618455366098295, + "loss": 2.5084, + "theoretical_loss": 3.4982417021109162, + "tokens_seen": 1589575680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618355065195587, + "loss": 2.5602, + "theoretical_loss": 3.498229127016418, + "tokens_seen": 1589641216 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618254764292879, + "loss": 2.6266, + "theoretical_loss": 3.498216552585496, + "tokens_seen": 1589706752 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026181544633901706, + "loss": 2.4609, + "theoretical_loss": 3.498203978818087, + "tokens_seen": 1589772288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618054162487463, + "loss": 2.2296, + "theoretical_loss": 3.49819140571413, + "tokens_seen": 1589837824 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617953861584754, + "loss": 2.5298, + "theoretical_loss": 3.4981788332735615, + "tokens_seen": 1589903360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026178535606820465, + "loss": 2.7224, + "theoretical_loss": 3.498166261496319, + "tokens_seen": 1589968896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617753259779338, + "loss": 2.8246, + "theoretical_loss": 3.498153690382341, + "tokens_seen": 1590034432 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261765295887663, + "loss": 2.6064, + "theoretical_loss": 3.4981411199315646, + "tokens_seen": 1590099968 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617552657973922, + "loss": 2.6017, + "theoretical_loss": 3.498128550143928, + "tokens_seen": 1590165504 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617452357071214, + "loss": 2.4292, + "theoretical_loss": 3.4981159810193683, + "tokens_seen": 1590231040 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026173520561685056, + "loss": 2.4612, + "theoretical_loss": 3.4981034125578234, + "tokens_seen": 1590296576 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026172517552657974, + "loss": 2.5168, + "theoretical_loss": 3.498090844759231, + "tokens_seen": 1590362112 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617151454363089, + "loss": 2.6873, + "theoretical_loss": 3.498078277623529, + "tokens_seen": 1590427648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026170511534603816, + "loss": 2.5425, + "theoretical_loss": 3.4980657111506552, + "tokens_seen": 1590493184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616950852557673, + "loss": 2.6458, + "theoretical_loss": 3.4980531453405463, + "tokens_seen": 1590558720 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616850551654965, + "loss": 2.3692, + "theoretical_loss": 3.4980405801931416, + "tokens_seen": 1590624256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026167502507522565, + "loss": 2.5229, + "theoretical_loss": 3.498028015708378, + "tokens_seen": 1590689792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616649949849549, + "loss": 2.5577, + "theoretical_loss": 3.498015451886193, + "tokens_seen": 1590755328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026165496489468406, + "loss": 2.6216, + "theoretical_loss": 3.4980028887265253, + "tokens_seen": 1590820864 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 879490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6622262001037598, + "objective/train/theoretical_loss": 3.4979903262293117, + "objective/train/tokens_used": 1611346400, + "theoretical_loss": 3.4979903262293117, + "tokens_seen": 1590886400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026164493480441324, + "loss": 2.6067, + "theoretical_loss": 3.4979903262293117, + "tokens_seen": 1590886400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616349047141424, + "loss": 2.6207, + "theoretical_loss": 3.4979777643944905, + "tokens_seen": 1590951936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026162487462387166, + "loss": 2.5697, + "theoretical_loss": 3.497965203221999, + "tokens_seen": 1591017472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616148445336008, + "loss": 2.7107, + "theoretical_loss": 3.497952642711776, + "tokens_seen": 1591083008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026160481444333, + "loss": 2.6887, + "theoretical_loss": 3.4979400828637583, + "tokens_seen": 1591148544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026159478435305915, + "loss": 2.4801, + "theoretical_loss": 3.497927523677884, + "tokens_seen": 1591214080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615847542627884, + "loss": 2.3117, + "theoretical_loss": 3.497914965154091, + "tokens_seen": 1591279616 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026157472417251757, + "loss": 2.4725, + "theoretical_loss": 3.4979024072923175, + "tokens_seen": 1591345152 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026156469408224675, + "loss": 2.6624, + "theoretical_loss": 3.4978898500925006, + "tokens_seen": 1591410688 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026155466399197593, + "loss": 2.5702, + "theoretical_loss": 3.4978772935545788, + "tokens_seen": 1591476224 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615446339017051, + "loss": 2.6589, + "theoretical_loss": 3.4978647376784897, + "tokens_seen": 1591541760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615346038114343, + "loss": 2.7033, + "theoretical_loss": 3.497852182464171, + "tokens_seen": 1591607296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615245737211635, + "loss": 2.5444, + "theoretical_loss": 3.4978396279115604, + "tokens_seen": 1591672832 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026151454363089265, + "loss": 2.4963, + "theoretical_loss": 3.497827074020597, + "tokens_seen": 1591738368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615045135406219, + "loss": 2.3806, + "theoretical_loss": 3.4978145207912172, + "tokens_seen": 1591803904 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261494483450351, + "loss": 2.6509, + "theoretical_loss": 3.497801968223359, + "tokens_seen": 1591869440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026148445336008025, + "loss": 2.3885, + "theoretical_loss": 3.4977894163169614, + "tokens_seen": 1591934976 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026147442326980943, + "loss": 2.6258, + "theoretical_loss": 3.4977768650719616, + "tokens_seen": 1592000512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614643931795386, + "loss": 2.8348, + "theoretical_loss": 3.4977643144882977, + "tokens_seen": 1592066048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614543630892678, + "loss": 2.6699, + "theoretical_loss": 3.4977517645659075, + "tokens_seen": 1592131584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026144433299899703, + "loss": 2.7163, + "theoretical_loss": 3.4977392153047298, + "tokens_seen": 1592197120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026143430290872616, + "loss": 2.6766, + "theoretical_loss": 3.497726666704701, + "tokens_seen": 1592262656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614242728184554, + "loss": 2.5892, + "theoretical_loss": 3.49771411876576, + "tokens_seen": 1592328192 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614142427281845, + "loss": 2.672, + "theoretical_loss": 3.4977015714878443, + "tokens_seen": 1592393728 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026140421263791375, + "loss": 2.6054, + "theoretical_loss": 3.4976890248708923, + "tokens_seen": 1592459264 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 880099, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.060887575149536, + "objective/train/theoretical_loss": 3.4976764789148422, + "objective/train/tokens_used": 1612984800, + "theoretical_loss": 3.4976764789148422, + "tokens_seen": 1592524800 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026139418254764293, + "loss": 2.8658, + "theoretical_loss": 3.4976764789148422, + "tokens_seen": 1592524800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613841524573721, + "loss": 2.648, + "theoretical_loss": 3.497663933619631, + "tokens_seen": 1592590336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613741223671013, + "loss": 2.7317, + "theoretical_loss": 3.4976513889851977, + "tokens_seen": 1592655872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613640922768305, + "loss": 2.6132, + "theoretical_loss": 3.4976388450114797, + "tokens_seen": 1592721408 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026135406218655966, + "loss": 2.5033, + "theoretical_loss": 3.497626301698416, + "tokens_seen": 1592786944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613440320962889, + "loss": 2.4996, + "theoretical_loss": 3.4976137590459433, + "tokens_seen": 1592852480 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261334002006018, + "loss": 2.3833, + "theoretical_loss": 3.497601217054, + "tokens_seen": 1592918016 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026132397191574726, + "loss": 2.5479, + "theoretical_loss": 3.497588675722525, + "tokens_seen": 1592983552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613139418254764, + "loss": 2.6199, + "theoretical_loss": 3.4975761350514554, + "tokens_seen": 1593049088 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613039117352056, + "loss": 2.7099, + "theoretical_loss": 3.4975635950407296, + "tokens_seen": 1593114624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612938816449348, + "loss": 2.4258, + "theoretical_loss": 3.497551055690286, + "tokens_seen": 1593180160 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261283851554664, + "loss": 2.8454, + "theoretical_loss": 3.4975385170000615, + "tokens_seen": 1593245696 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026127382146439316, + "loss": 2.5293, + "theoretical_loss": 3.4975259789699953, + "tokens_seen": 1593311232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612637913741224, + "loss": 2.6926, + "theoretical_loss": 3.4975134416000255, + "tokens_seen": 1593376768 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612537612838516, + "loss": 2.6239, + "theoretical_loss": 3.4975009048900896, + "tokens_seen": 1593442304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026124373119358076, + "loss": 2.5472, + "theoretical_loss": 3.497488368840126, + "tokens_seen": 1593507840 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026123370110330994, + "loss": 2.6316, + "theoretical_loss": 3.4974758334500726, + "tokens_seen": 1593573376 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612236710130391, + "loss": 2.5666, + "theoretical_loss": 3.497463298719868, + "tokens_seen": 1593638912 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026121364092276836, + "loss": 2.8978, + "theoretical_loss": 3.49745076464945, + "tokens_seen": 1593704448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612036108324975, + "loss": 2.622, + "theoretical_loss": 3.4974382312387573, + "tokens_seen": 1593769984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611935807422267, + "loss": 2.5524, + "theoretical_loss": 3.497425698487727, + "tokens_seen": 1593835520 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026118355065195585, + "loss": 2.4529, + "theoretical_loss": 3.497413166396298, + "tokens_seen": 1593901056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611735205616851, + "loss": 2.5538, + "theoretical_loss": 3.497400634964408, + "tokens_seen": 1593966592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026116349047141426, + "loss": 2.4304, + "theoretical_loss": 3.497388104191996, + "tokens_seen": 1594032128 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026115346038114344, + "loss": 2.651, + "theoretical_loss": 3.4973755740789993, + "tokens_seen": 1594097664 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 881131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6384732723236084, + "objective/train/theoretical_loss": 3.4973630446253563, + "objective/train/tokens_used": 1614623200, + "theoretical_loss": 3.4973630446253563, + "tokens_seen": 1594163200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611434302908726, + "loss": 2.5996, + "theoretical_loss": 3.4973630446253563, + "tokens_seen": 1594163200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026113340020060186, + "loss": 2.7237, + "theoretical_loss": 3.4973505158310054, + "tokens_seen": 1594228736 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261123370110331, + "loss": 2.7224, + "theoretical_loss": 3.497337987695885, + "tokens_seen": 1594294272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611133400200602, + "loss": 2.4421, + "theoretical_loss": 3.4973254602199333, + "tokens_seen": 1594359808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026110330992978935, + "loss": 2.4702, + "theoretical_loss": 3.4973129334030877, + "tokens_seen": 1594425344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610932798395186, + "loss": 2.7693, + "theoretical_loss": 3.4973004072452873, + "tokens_seen": 1594490880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026108324974924777, + "loss": 2.5035, + "theoretical_loss": 3.4972878817464697, + "tokens_seen": 1594556416 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026107321965897695, + "loss": 2.7898, + "theoretical_loss": 3.4972753569065738, + "tokens_seen": 1594621952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026106318956870613, + "loss": 2.5371, + "theoretical_loss": 3.4972628327255375, + "tokens_seen": 1594687488 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610531594784353, + "loss": 2.617, + "theoretical_loss": 3.4972503092032996, + "tokens_seen": 1594753024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610431293881645, + "loss": 2.4432, + "theoretical_loss": 3.497237786339797, + "tokens_seen": 1594818560 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610330992978937, + "loss": 2.5231, + "theoretical_loss": 3.4972252641349693, + "tokens_seen": 1594884096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026102306920762285, + "loss": 2.6498, + "theoretical_loss": 3.4972127425887547, + "tokens_seen": 1594949632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610130391173521, + "loss": 2.6556, + "theoretical_loss": 3.4972002217010907, + "tokens_seen": 1595015168 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610030090270812, + "loss": 2.5335, + "theoretical_loss": 3.4971877014719164, + "tokens_seen": 1595080704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026099297893681045, + "loss": 2.7581, + "theoretical_loss": 3.49717518190117, + "tokens_seen": 1595146240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026098294884653963, + "loss": 2.5752, + "theoretical_loss": 3.4971626629887895, + "tokens_seen": 1595211776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002609729187562688, + "loss": 2.8398, + "theoretical_loss": 3.497150144734713, + "tokens_seen": 1595277312 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260962888665998, + "loss": 2.5584, + "theoretical_loss": 3.49713762713888, + "tokens_seen": 1595342848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026095285857572723, + "loss": 2.6581, + "theoretical_loss": 3.4971251102012273, + "tokens_seen": 1595408384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026094282848545636, + "loss": 2.3576, + "theoretical_loss": 3.4971125939216936, + "tokens_seen": 1595473920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002609327983951856, + "loss": 2.801, + "theoretical_loss": 3.4971000783002184, + "tokens_seen": 1595539456 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002609227683049147, + "loss": 2.613, + "theoretical_loss": 3.4970875633367395, + "tokens_seen": 1595604992 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026091273821464395, + "loss": 2.5997, + "theoretical_loss": 3.497075049031195, + "tokens_seen": 1595670528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026090270812437313, + "loss": 2.4345, + "theoretical_loss": 3.497062535383523, + "tokens_seen": 1595736064 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 881793, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.461489677429199, + "objective/train/theoretical_loss": 3.4970500223936627, + "objective/train/tokens_used": 1616261600, + "theoretical_loss": 3.4970500223936627, + "tokens_seen": 1595801600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608926780341023, + "loss": 2.7317, + "theoretical_loss": 3.4970500223936627, + "tokens_seen": 1595801600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608826479438315, + "loss": 2.2827, + "theoretical_loss": 3.4970375100615523, + "tokens_seen": 1595867136 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608726178535607, + "loss": 2.855, + "theoretical_loss": 3.49702499838713, + "tokens_seen": 1595932672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026086258776328986, + "loss": 2.6554, + "theoretical_loss": 3.497012487370334, + "tokens_seen": 1595998208 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608525576730191, + "loss": 2.4999, + "theoretical_loss": 3.496999977011103, + "tokens_seen": 1596063744 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608425275827482, + "loss": 2.4595, + "theoretical_loss": 3.4969874673093755, + "tokens_seen": 1596129280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026083249749247746, + "loss": 2.7592, + "theoretical_loss": 3.49697495826509, + "tokens_seen": 1596194816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608224674022066, + "loss": 2.752, + "theoretical_loss": 3.496962449878185, + "tokens_seen": 1596260352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608124373119358, + "loss": 2.4624, + "theoretical_loss": 3.4969499421485986, + "tokens_seen": 1596325888 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260802407221665, + "loss": 2.5857, + "theoretical_loss": 3.496937435076269, + "tokens_seen": 1596391424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607923771313942, + "loss": 2.7426, + "theoretical_loss": 3.4969249286611364, + "tokens_seen": 1596456960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026078234704112336, + "loss": 2.6939, + "theoretical_loss": 3.4969124229031374, + "tokens_seen": 1596522496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607723169508526, + "loss": 2.6724, + "theoretical_loss": 3.496899917802211, + "tokens_seen": 1596588032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607622868605817, + "loss": 2.6715, + "theoretical_loss": 3.496887413358296, + "tokens_seen": 1596653568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026075225677031096, + "loss": 2.6495, + "theoretical_loss": 3.4968749095713307, + "tokens_seen": 1596719104 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607422266800401, + "loss": 2.4519, + "theoretical_loss": 3.496862406441254, + "tokens_seen": 1596784640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607321965897693, + "loss": 2.362, + "theoretical_loss": 3.4968499039680037, + "tokens_seen": 1596850176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607221664994985, + "loss": 2.5348, + "theoretical_loss": 3.4968374021515194, + "tokens_seen": 1596915712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607121364092277, + "loss": 2.9297, + "theoretical_loss": 3.4968249009917383, + "tokens_seen": 1596981248 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026070210631895687, + "loss": 2.5659, + "theoretical_loss": 3.4968124004886003, + "tokens_seen": 1597046784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026069207622868605, + "loss": 2.5782, + "theoretical_loss": 3.496799900642043, + "tokens_seen": 1597112320 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026068204613841523, + "loss": 2.5968, + "theoretical_loss": 3.4967874014520057, + "tokens_seen": 1597177856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026067201604814446, + "loss": 2.4417, + "theoretical_loss": 3.4967749029184265, + "tokens_seen": 1597243392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606619859578736, + "loss": 2.5994, + "theoretical_loss": 3.4967624050412436, + "tokens_seen": 1597308928 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606519558676028, + "loss": 2.4952, + "theoretical_loss": 3.496749907820397, + "tokens_seen": 1597374464 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 882868, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.732029914855957, + "objective/train/theoretical_loss": 3.4967374112558236, + "objective/train/tokens_used": 1617900000, + "theoretical_loss": 3.4967374112558236, + "tokens_seen": 1597440000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026064192577733195, + "loss": 2.4951, + "theoretical_loss": 3.4967374112558236, + "tokens_seen": 1597440000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606318956870612, + "loss": 2.6173, + "theoretical_loss": 3.496724915347463, + "tokens_seen": 1597505536 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026062186559679037, + "loss": 2.4632, + "theoretical_loss": 3.496712420095254, + "tokens_seen": 1597571072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026061183550651955, + "loss": 2.6571, + "theoretical_loss": 3.4966999254991347, + "tokens_seen": 1597636608 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026060180541624873, + "loss": 2.6554, + "theoretical_loss": 3.4966874315590437, + "tokens_seen": 1597702144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026059177532597797, + "loss": 2.6584, + "theoretical_loss": 3.49667493827492, + "tokens_seen": 1597767680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605817452357071, + "loss": 2.771, + "theoretical_loss": 3.4966624456467024, + "tokens_seen": 1597833216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026057171514543633, + "loss": 2.6361, + "theoretical_loss": 3.496649953674329, + "tokens_seen": 1597898752 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026056168505516546, + "loss": 2.6624, + "theoretical_loss": 3.4966374623577385, + "tokens_seen": 1597964288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605516549648947, + "loss": 2.666, + "theoretical_loss": 3.4966249716968703, + "tokens_seen": 1598029824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026054162487462387, + "loss": 2.6346, + "theoretical_loss": 3.4966124816916624, + "tokens_seen": 1598095360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026053159478435305, + "loss": 2.5328, + "theoretical_loss": 3.496599992342054, + "tokens_seen": 1598160896 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026052156469408223, + "loss": 2.5264, + "theoretical_loss": 3.496587503647983, + "tokens_seen": 1598226432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605115346038114, + "loss": 2.6111, + "theoretical_loss": 3.4965750156093893, + "tokens_seen": 1598291968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026050150451354065, + "loss": 2.5371, + "theoretical_loss": 3.4965625282262107, + "tokens_seen": 1598357504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026049147442326983, + "loss": 2.6252, + "theoretical_loss": 3.496550041498386, + "tokens_seen": 1598423040 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260481444332999, + "loss": 2.8174, + "theoretical_loss": 3.496537555425854, + "tokens_seen": 1598488576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604714142427282, + "loss": 2.7547, + "theoretical_loss": 3.496525070008554, + "tokens_seen": 1598554112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026046138415245743, + "loss": 2.6282, + "theoretical_loss": 3.4965125852464243, + "tokens_seen": 1598619648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026045135406218656, + "loss": 2.4625, + "theoretical_loss": 3.4965001011394037, + "tokens_seen": 1598685184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604413239719158, + "loss": 2.4502, + "theoretical_loss": 3.496487617687431, + "tokens_seen": 1598750720 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604312938816449, + "loss": 2.6997, + "theoretical_loss": 3.496475134890445, + "tokens_seen": 1598816256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026042126379137415, + "loss": 2.6514, + "theoretical_loss": 3.496462652748384, + "tokens_seen": 1598881792 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026041123370110333, + "loss": 2.3242, + "theoretical_loss": 3.4964501712611873, + "tokens_seen": 1598947328 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604012036108325, + "loss": 2.7017, + "theoretical_loss": 3.4964376904287935, + "tokens_seen": 1599012864 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 883378, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9928767681121826, + "objective/train/theoretical_loss": 3.496425210251142, + "objective/train/tokens_used": 1619538400, + "theoretical_loss": 3.496425210251142, + "tokens_seen": 1599078400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603911735205617, + "loss": 2.5989, + "theoretical_loss": 3.496425210251142, + "tokens_seen": 1599078400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603811434302909, + "loss": 2.6122, + "theoretical_loss": 3.496412730728171, + "tokens_seen": 1599143936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026037111334002006, + "loss": 2.5824, + "theoretical_loss": 3.4964002518598196, + "tokens_seen": 1599209472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603610832497493, + "loss": 2.4081, + "theoretical_loss": 3.4963877736460267, + "tokens_seen": 1599275008 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603510531594784, + "loss": 2.474, + "theoretical_loss": 3.4963752960867307, + "tokens_seen": 1599340544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026034102306920766, + "loss": 2.5778, + "theoretical_loss": 3.496362819181871, + "tokens_seen": 1599406080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603309929789368, + "loss": 2.6896, + "theoretical_loss": 3.496350342931386, + "tokens_seen": 1599471616 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260320962888666, + "loss": 2.5757, + "theoretical_loss": 3.496337867335215, + "tokens_seen": 1599537152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603109327983952, + "loss": 2.7654, + "theoretical_loss": 3.496325392393296, + "tokens_seen": 1599602688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603009027081244, + "loss": 2.4686, + "theoretical_loss": 3.496312918105569, + "tokens_seen": 1599668224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026029087261785356, + "loss": 2.6196, + "theoretical_loss": 3.4963004444719723, + "tokens_seen": 1599733760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602808425275828, + "loss": 2.4787, + "theoretical_loss": 3.4962879714924453, + "tokens_seen": 1599799296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602708124373119, + "loss": 2.6034, + "theoretical_loss": 3.4962754991669263, + "tokens_seen": 1599864832 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026026078234704116, + "loss": 2.7047, + "theoretical_loss": 3.4962630274953543, + "tokens_seen": 1599930368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602507522567703, + "loss": 2.7864, + "theoretical_loss": 3.4962505564776682, + "tokens_seen": 1599995904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602407221664995, + "loss": 2.6546, + "theoretical_loss": 3.4962380861138076, + "tokens_seen": 1600061440 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602306920762287, + "loss": 2.5206, + "theoretical_loss": 3.496225616403711, + "tokens_seen": 1600126976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602206619859579, + "loss": 2.5829, + "theoretical_loss": 3.496213147347317, + "tokens_seen": 1600192512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026021063189568707, + "loss": 2.5875, + "theoretical_loss": 3.496200678944565, + "tokens_seen": 1600258048 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026020060180541625, + "loss": 2.6611, + "theoretical_loss": 3.4961882111953937, + "tokens_seen": 1600323584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026019057171514543, + "loss": 2.6666, + "theoretical_loss": 3.4961757440997427, + "tokens_seen": 1600389120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026018054162487466, + "loss": 2.5267, + "theoretical_loss": 3.49616327765755, + "tokens_seen": 1600454656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002601705115346038, + "loss": 2.4489, + "theoretical_loss": 3.4961508118687554, + "tokens_seen": 1600520192 + }, + { + "epoch": 0.49, + "learning_rate": 0.000260160481444333, + "loss": 2.5302, + "theoretical_loss": 3.4961383467332974, + "tokens_seen": 1600585728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026015045135406215, + "loss": 2.6217, + "theoretical_loss": 3.4961258822511154, + "tokens_seen": 1600651264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 884433, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5312654972076416, + "objective/train/theoretical_loss": 3.4961134184221483, + "objective/train/tokens_used": 1621176800, + "theoretical_loss": 3.4961134184221483, + "tokens_seen": 1600716800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002601404212637914, + "loss": 2.5328, + "theoretical_loss": 3.4961134184221483, + "tokens_seen": 1600716800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026013039117352057, + "loss": 2.5875, + "theoretical_loss": 3.4961009552463347, + "tokens_seen": 1600782336 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026012036108324975, + "loss": 2.8462, + "theoretical_loss": 3.496088492723614, + "tokens_seen": 1600847872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026011033099297893, + "loss": 2.5826, + "theoretical_loss": 3.496076030853926, + "tokens_seen": 1600913408 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026010030090270817, + "loss": 2.5677, + "theoretical_loss": 3.496063569637208, + "tokens_seen": 1600978944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600902708124373, + "loss": 2.6, + "theoretical_loss": 3.4960511090734006, + "tokens_seen": 1601044480 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026008024072216653, + "loss": 2.6757, + "theoretical_loss": 3.4960386491624424, + "tokens_seen": 1601110016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026007021063189566, + "loss": 2.6101, + "theoretical_loss": 3.4960261899042724, + "tokens_seen": 1601175552 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600601805416249, + "loss": 2.674, + "theoretical_loss": 3.496013731298829, + "tokens_seen": 1601241088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026005015045135407, + "loss": 2.7099, + "theoretical_loss": 3.496001273346053, + "tokens_seen": 1601306624 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026004012036108325, + "loss": 2.3682, + "theoretical_loss": 3.4959888160458816, + "tokens_seen": 1601372160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026003009027081243, + "loss": 2.6105, + "theoretical_loss": 3.495976359398255, + "tokens_seen": 1601437696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600200601805416, + "loss": 2.49, + "theoretical_loss": 3.495963903403112, + "tokens_seen": 1601503232 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600100300902708, + "loss": 2.3433, + "theoretical_loss": 3.495951448060392, + "tokens_seen": 1601568768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026000000000000003, + "loss": 2.697, + "theoretical_loss": 3.495938993370034, + "tokens_seen": 1601634304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025998996990972916, + "loss": 2.6611, + "theoretical_loss": 3.495926539331977, + "tokens_seen": 1601699840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599799398194584, + "loss": 2.6991, + "theoretical_loss": 3.4959140859461604, + "tokens_seen": 1601765376 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599699097291876, + "loss": 2.5537, + "theoretical_loss": 3.4959016332125232, + "tokens_seen": 1601830912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025995987963891676, + "loss": 2.6926, + "theoretical_loss": 3.4958891811310044, + "tokens_seen": 1601896448 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025994984954864594, + "loss": 2.6165, + "theoretical_loss": 3.495876729701543, + "tokens_seen": 1601961984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599398194583751, + "loss": 2.6984, + "theoretical_loss": 3.495864278924079, + "tokens_seen": 1602027520 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599297893681043, + "loss": 2.4181, + "theoretical_loss": 3.4958518287985507, + "tokens_seen": 1602093056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025991975927783353, + "loss": 2.6084, + "theoretical_loss": 3.495839379324898, + "tokens_seen": 1602158592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025990972918756266, + "loss": 2.8353, + "theoretical_loss": 3.4958269305030596, + "tokens_seen": 1602224128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598996990972919, + "loss": 2.5644, + "theoretical_loss": 3.4958144823329755, + "tokens_seen": 1602289664 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 884978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.275560140609741, + "objective/train/theoretical_loss": 3.4958020348145835, + "objective/train/tokens_used": 1622815200, + "theoretical_loss": 3.4958020348145835, + "tokens_seen": 1602355200 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259889669007021, + "loss": 2.4181, + "theoretical_loss": 3.4958020348145835, + "tokens_seen": 1602355200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025987963891675026, + "loss": 2.707, + "theoretical_loss": 3.495789587947824, + "tokens_seen": 1602420736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025986960882647944, + "loss": 2.4409, + "theoretical_loss": 3.4957771417326358, + "tokens_seen": 1602486272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598595787362086, + "loss": 2.5454, + "theoretical_loss": 3.4957646961689584, + "tokens_seen": 1602551808 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598495486459378, + "loss": 2.5064, + "theoretical_loss": 3.4957522512567314, + "tokens_seen": 1602617344 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259839518555667, + "loss": 2.8227, + "theoretical_loss": 3.4957398069958927, + "tokens_seen": 1602682880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025982948846539616, + "loss": 2.6371, + "theoretical_loss": 3.495727363386383, + "tokens_seen": 1602748416 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598194583751254, + "loss": 2.4321, + "theoretical_loss": 3.495714920428141, + "tokens_seen": 1602813952 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598094282848545, + "loss": 2.4335, + "theoretical_loss": 3.495702478121106, + "tokens_seen": 1602879488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025979939819458376, + "loss": 2.7261, + "theoretical_loss": 3.4956900364652164, + "tokens_seen": 1602945024 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025978936810431294, + "loss": 2.6027, + "theoretical_loss": 3.4956775954604136, + "tokens_seen": 1603010560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597793380140421, + "loss": 2.4884, + "theoretical_loss": 3.495665155106635, + "tokens_seen": 1603076096 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597693079237713, + "loss": 2.7455, + "theoretical_loss": 3.4956527154038213, + "tokens_seen": 1603141632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597592778335005, + "loss": 2.6733, + "theoretical_loss": 3.4956402763519105, + "tokens_seen": 1603207168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597492477432297, + "loss": 2.5342, + "theoretical_loss": 3.4956278379508428, + "tokens_seen": 1603272704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597392176529589, + "loss": 2.6573, + "theoretical_loss": 3.4956154002005575, + "tokens_seen": 1603338240 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597291875626881, + "loss": 2.5731, + "theoretical_loss": 3.4956029631009935, + "tokens_seen": 1603403776 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025971915747241727, + "loss": 2.6231, + "theoretical_loss": 3.495590526652091, + "tokens_seen": 1603469312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025970912738214645, + "loss": 2.6964, + "theoretical_loss": 3.4955780908537886, + "tokens_seen": 1603534848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025969909729187563, + "loss": 2.6514, + "theoretical_loss": 3.4955656557060255, + "tokens_seen": 1603600384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025968906720160486, + "loss": 2.5183, + "theoretical_loss": 3.4955532212087417, + "tokens_seen": 1603665920 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259679037111334, + "loss": 2.7161, + "theoretical_loss": 3.4955407873618762, + "tokens_seen": 1603731456 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002596690070210632, + "loss": 2.5747, + "theoretical_loss": 3.4955283541653692, + "tokens_seen": 1603796992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025965897693079235, + "loss": 2.3093, + "theoretical_loss": 3.495515921619159, + "tokens_seen": 1603862528 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002596489468405216, + "loss": 2.5467, + "theoretical_loss": 3.495503489723186, + "tokens_seen": 1603928064 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 886048, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6932458877563477, + "objective/train/theoretical_loss": 3.4954910584773886, + "objective/train/tokens_used": 1624453600, + "theoretical_loss": 3.4954910584773886, + "tokens_seen": 1603993600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025963891675025077, + "loss": 2.5246, + "theoretical_loss": 3.4954910584773886, + "tokens_seen": 1603993600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025962888665997995, + "loss": 2.4522, + "theoretical_loss": 3.495478627881707, + "tokens_seen": 1604059136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025961885656970913, + "loss": 2.7995, + "theoretical_loss": 3.4954661979360804, + "tokens_seen": 1604124672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025960882647943837, + "loss": 2.6769, + "theoretical_loss": 3.4954537686404485, + "tokens_seen": 1604190208 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595987963891675, + "loss": 2.7504, + "theoretical_loss": 3.49544133999475, + "tokens_seen": 1604255744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025958876629889673, + "loss": 2.5586, + "theoretical_loss": 3.4954289119989252, + "tokens_seen": 1604321280 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025957873620862586, + "loss": 2.6534, + "theoretical_loss": 3.495416484652913, + "tokens_seen": 1604386816 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595687061183551, + "loss": 2.5472, + "theoretical_loss": 3.4954040579566534, + "tokens_seen": 1604452352 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025955867602808427, + "loss": 2.5427, + "theoretical_loss": 3.4953916319100857, + "tokens_seen": 1604517888 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025954864593781345, + "loss": 2.5922, + "theoretical_loss": 3.4953792065131495, + "tokens_seen": 1604583424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025953861584754263, + "loss": 2.7647, + "theoretical_loss": 3.4953667817657834, + "tokens_seen": 1604648960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595285857572718, + "loss": 2.6222, + "theoretical_loss": 3.495354357667928, + "tokens_seen": 1604714496 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259518555667001, + "loss": 2.6812, + "theoretical_loss": 3.495341934219523, + "tokens_seen": 1604780032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025950852557673023, + "loss": 2.5888, + "theoretical_loss": 3.4953295114205067, + "tokens_seen": 1604845568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025949849548645936, + "loss": 2.6561, + "theoretical_loss": 3.4953170892708196, + "tokens_seen": 1604911104 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594884653961886, + "loss": 2.521, + "theoretical_loss": 3.495304667770401, + "tokens_seen": 1604976640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594784353059178, + "loss": 2.6409, + "theoretical_loss": 3.4952922469191905, + "tokens_seen": 1605042176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025946840521564696, + "loss": 2.6746, + "theoretical_loss": 3.4952798267171277, + "tokens_seen": 1605107712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025945837512537614, + "loss": 2.6635, + "theoretical_loss": 3.495267407164152, + "tokens_seen": 1605173248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594483450351053, + "loss": 2.5861, + "theoretical_loss": 3.4952549882602026, + "tokens_seen": 1605238784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594383149448345, + "loss": 2.7291, + "theoretical_loss": 3.49524257000522, + "tokens_seen": 1605304320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025942828485456373, + "loss": 2.6775, + "theoretical_loss": 3.4952301523991434, + "tokens_seen": 1605369856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025941825476429286, + "loss": 2.6408, + "theoretical_loss": 3.495217735441912, + "tokens_seen": 1605435392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594082246740221, + "loss": 2.4916, + "theoretical_loss": 3.495205319133466, + "tokens_seen": 1605500928 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593981945837512, + "loss": 2.5286, + "theoretical_loss": 3.495192903473745, + "tokens_seen": 1605566464 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 887036, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7378838062286377, + "objective/train/theoretical_loss": 3.4951804884626885, + "objective/train/tokens_used": 1626092000, + "theoretical_loss": 3.4951804884626885, + "tokens_seen": 1605632000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025938816449348046, + "loss": 2.5401, + "theoretical_loss": 3.4951804884626885, + "tokens_seen": 1605632000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025937813440320964, + "loss": 2.8037, + "theoretical_loss": 3.4951680741002358, + "tokens_seen": 1605697536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593681043129388, + "loss": 2.4051, + "theoretical_loss": 3.4951556603863265, + "tokens_seen": 1605763072 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259358074222668, + "loss": 2.6145, + "theoretical_loss": 3.4951432473209003, + "tokens_seen": 1605828608 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593480441323972, + "loss": 2.5647, + "theoretical_loss": 3.4951308349038976, + "tokens_seen": 1605894144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025933801404212636, + "loss": 2.636, + "theoretical_loss": 3.4951184231352577, + "tokens_seen": 1605959680 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593279839518556, + "loss": 2.3618, + "theoretical_loss": 3.49510601201492, + "tokens_seen": 1606025216 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593179538615847, + "loss": 2.5, + "theoretical_loss": 3.4950936015428242, + "tokens_seen": 1606090752 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025930792377131396, + "loss": 2.5412, + "theoretical_loss": 3.49508119171891, + "tokens_seen": 1606156288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025929789368104314, + "loss": 2.4135, + "theoretical_loss": 3.4950687825431177, + "tokens_seen": 1606221824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592878635907723, + "loss": 2.5794, + "theoretical_loss": 3.4950563740153866, + "tokens_seen": 1606287360 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592778335005015, + "loss": 2.6096, + "theoretical_loss": 3.495043966135656, + "tokens_seen": 1606352896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592678034102307, + "loss": 2.6201, + "theoretical_loss": 3.4950315589038654, + "tokens_seen": 1606418432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025925777331995987, + "loss": 2.5927, + "theoretical_loss": 3.495019152319956, + "tokens_seen": 1606483968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592477432296891, + "loss": 2.7788, + "theoretical_loss": 3.4950067463838663, + "tokens_seen": 1606549504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025923771313941823, + "loss": 2.6531, + "theoretical_loss": 3.4949943410955364, + "tokens_seen": 1606615040 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025922768304914747, + "loss": 2.4365, + "theoretical_loss": 3.494981936454906, + "tokens_seen": 1606680576 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592176529588766, + "loss": 2.5671, + "theoretical_loss": 3.494969532461915, + "tokens_seen": 1606746112 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025920762286860583, + "loss": 2.5943, + "theoretical_loss": 3.494957129116503, + "tokens_seen": 1606811648 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259197592778335, + "loss": 2.4938, + "theoretical_loss": 3.49494472641861, + "tokens_seen": 1606877184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591875626880642, + "loss": 2.6406, + "theoretical_loss": 3.494932324368176, + "tokens_seen": 1606942720 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025917753259779337, + "loss": 2.7251, + "theoretical_loss": 3.49491992296514, + "tokens_seen": 1607008256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025916750250752255, + "loss": 2.5622, + "theoretical_loss": 3.4949075222094423, + "tokens_seen": 1607073792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025915747241725173, + "loss": 2.62, + "theoretical_loss": 3.4948951221010227, + "tokens_seen": 1607139328 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025914744232698097, + "loss": 2.6759, + "theoretical_loss": 3.494882722639821, + "tokens_seen": 1607204864 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 887423, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.676961898803711, + "objective/train/theoretical_loss": 3.494870323825777, + "objective/train/tokens_used": 1627730400, + "theoretical_loss": 3.494870323825777, + "tokens_seen": 1607270400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591374122367101, + "loss": 2.7373, + "theoretical_loss": 3.494870323825777, + "tokens_seen": 1607270400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025912738214643933, + "loss": 2.7514, + "theoretical_loss": 3.494857925658831, + "tokens_seen": 1607335936 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591173520561685, + "loss": 2.8276, + "theoretical_loss": 3.4948455281389217, + "tokens_seen": 1607401472 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591073219658977, + "loss": 2.5974, + "theoretical_loss": 3.49483313126599, + "tokens_seen": 1607467008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590972918756269, + "loss": 2.5221, + "theoretical_loss": 3.4948207350399754, + "tokens_seen": 1607532544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025908726178535606, + "loss": 2.4486, + "theoretical_loss": 3.494808339460818, + "tokens_seen": 1607598080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025907723169508524, + "loss": 2.3917, + "theoretical_loss": 3.494795944528457, + "tokens_seen": 1607663616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025906720160481447, + "loss": 2.606, + "theoretical_loss": 3.4947835502428335, + "tokens_seen": 1607729152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590571715145436, + "loss": 2.6469, + "theoretical_loss": 3.4947711566038864, + "tokens_seen": 1607794688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025904714142427283, + "loss": 2.762, + "theoretical_loss": 3.4947587636115554, + "tokens_seen": 1607860224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025903711133400196, + "loss": 2.7188, + "theoretical_loss": 3.494746371265781, + "tokens_seen": 1607925760 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590270812437312, + "loss": 2.6411, + "theoretical_loss": 3.494733979566503, + "tokens_seen": 1607991296 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590170511534604, + "loss": 2.3122, + "theoretical_loss": 3.494721588513662, + "tokens_seen": 1608056832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025900702106318956, + "loss": 2.571, + "theoretical_loss": 3.4947091981071963, + "tokens_seen": 1608122368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589969909729188, + "loss": 2.7087, + "theoretical_loss": 3.4946968083470473, + "tokens_seen": 1608187904 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258986960882648, + "loss": 2.4094, + "theoretical_loss": 3.4946844192331543, + "tokens_seen": 1608253440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025897693079237716, + "loss": 2.7878, + "theoretical_loss": 3.4946720307654573, + "tokens_seen": 1608318976 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025896690070210634, + "loss": 2.6973, + "theoretical_loss": 3.4946596429438967, + "tokens_seen": 1608384512 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589568706118355, + "loss": 2.8017, + "theoretical_loss": 3.494647255768412, + "tokens_seen": 1608450048 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589468405215647, + "loss": 2.8646, + "theoretical_loss": 3.494634869238943, + "tokens_seen": 1608515584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025893681043129393, + "loss": 2.4821, + "theoretical_loss": 3.4946224833554305, + "tokens_seen": 1608581120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025892678034102306, + "loss": 2.5314, + "theoretical_loss": 3.4946100981178136, + "tokens_seen": 1608646656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589167502507523, + "loss": 2.5069, + "theoretical_loss": 3.4945977135260327, + "tokens_seen": 1608712192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589067201604814, + "loss": 2.7645, + "theoretical_loss": 3.494585329580028, + "tokens_seen": 1608777728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025889669007021066, + "loss": 2.6254, + "theoretical_loss": 3.4945729462797397, + "tokens_seen": 1608843264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 888663, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4536147117614746, + "objective/train/theoretical_loss": 3.494560563625107, + "objective/train/tokens_used": 1629368800, + "theoretical_loss": 3.494560563625107, + "tokens_seen": 1608908800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025888665997993984, + "loss": 2.7273, + "theoretical_loss": 3.494560563625107, + "tokens_seen": 1608908800 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258876629889669, + "loss": 2.778, + "theoretical_loss": 3.4945481816160706, + "tokens_seen": 1608974336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588665997993982, + "loss": 2.7873, + "theoretical_loss": 3.4945358002525704, + "tokens_seen": 1609039872 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588565697091274, + "loss": 2.6672, + "theoretical_loss": 3.4945234195345463, + "tokens_seen": 1609105408 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025884653961885657, + "loss": 2.6069, + "theoretical_loss": 3.4945110394619388, + "tokens_seen": 1609170944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588365095285858, + "loss": 2.408, + "theoretical_loss": 3.494498660034687, + "tokens_seen": 1609236480 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025882647943831493, + "loss": 2.4127, + "theoretical_loss": 3.4944862812527324, + "tokens_seen": 1609302016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025881644934804416, + "loss": 2.5735, + "theoretical_loss": 3.4944739031160137, + "tokens_seen": 1609367552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025880641925777334, + "loss": 2.5958, + "theoretical_loss": 3.4944615256244718, + "tokens_seen": 1609433088 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587963891675025, + "loss": 2.7475, + "theoretical_loss": 3.4944491487780467, + "tokens_seen": 1609498624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587863590772317, + "loss": 2.6384, + "theoretical_loss": 3.4944367725766785, + "tokens_seen": 1609564160 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587763289869609, + "loss": 2.5391, + "theoretical_loss": 3.494424397020307, + "tokens_seen": 1609629696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025876629889669007, + "loss": 2.5014, + "theoretical_loss": 3.4944120221088726, + "tokens_seen": 1609695232 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587562688064193, + "loss": 2.5073, + "theoretical_loss": 3.4943996478423154, + "tokens_seen": 1609760768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025874623871614843, + "loss": 2.7434, + "theoretical_loss": 3.4943872742205757, + "tokens_seen": 1609826304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025873620862587767, + "loss": 2.4657, + "theoretical_loss": 3.4943749012435936, + "tokens_seen": 1609891840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587261785356068, + "loss": 2.7081, + "theoretical_loss": 3.494362528911309, + "tokens_seen": 1609957376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025871614844533603, + "loss": 2.7731, + "theoretical_loss": 3.4943501572236624, + "tokens_seen": 1610022912 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587061183550652, + "loss": 2.6584, + "theoretical_loss": 3.4943377861805933, + "tokens_seen": 1610088448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586960882647944, + "loss": 2.63, + "theoretical_loss": 3.494325415782043, + "tokens_seen": 1610153984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025868605817452357, + "loss": 2.4182, + "theoretical_loss": 3.494313046027951, + "tokens_seen": 1610219520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025867602808425275, + "loss": 2.5206, + "theoretical_loss": 3.4943006769182574, + "tokens_seen": 1610285056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025866599799398193, + "loss": 2.5546, + "theoretical_loss": 3.4942883084529024, + "tokens_seen": 1610350592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025865596790371117, + "loss": 2.5437, + "theoretical_loss": 3.4942759406318262, + "tokens_seen": 1610416128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586459378134403, + "loss": 2.6966, + "theoretical_loss": 3.49426357345497, + "tokens_seen": 1610481664 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 889165, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2135186195373535, + "objective/train/theoretical_loss": 3.4942512069222724, + "objective/train/tokens_used": 1631007200, + "theoretical_loss": 3.4942512069222724, + "tokens_seen": 1610547200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025863590772316953, + "loss": 2.4609, + "theoretical_loss": 3.4942512069222724, + "tokens_seen": 1610547200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586258776328987, + "loss": 2.51, + "theoretical_loss": 3.494238841033675, + "tokens_seen": 1610612736 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586158475426279, + "loss": 2.4031, + "theoretical_loss": 3.4942264757891173, + "tokens_seen": 1610678272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586058174523571, + "loss": 2.9833, + "theoretical_loss": 3.4942141111885396, + "tokens_seen": 1610743808 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025859578736208626, + "loss": 2.598, + "theoretical_loss": 3.494201747231883, + "tokens_seen": 1610809344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025858575727181544, + "loss": 2.836, + "theoretical_loss": 3.4941893839190863, + "tokens_seen": 1610874880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025857572718154467, + "loss": 2.7011, + "theoretical_loss": 3.494177021250091, + "tokens_seen": 1610940416 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585656970912738, + "loss": 2.6018, + "theoretical_loss": 3.494164659224837, + "tokens_seen": 1611005952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025855566700100303, + "loss": 2.5364, + "theoretical_loss": 3.4941522978432644, + "tokens_seen": 1611071488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025854563691073216, + "loss": 2.8671, + "theoretical_loss": 3.4941399371053143, + "tokens_seen": 1611137024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585356068204614, + "loss": 2.569, + "theoretical_loss": 3.494127577010926, + "tokens_seen": 1611202560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585255767301906, + "loss": 2.7617, + "theoretical_loss": 3.49411521756004, + "tokens_seen": 1611268096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025851554663991976, + "loss": 2.3829, + "theoretical_loss": 3.494102858752597, + "tokens_seen": 1611333632 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025850551654964894, + "loss": 2.6156, + "theoretical_loss": 3.4940905005885368, + "tokens_seen": 1611399168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584954864593782, + "loss": 2.8408, + "theoretical_loss": 3.4940781430678003, + "tokens_seen": 1611464704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584854563691073, + "loss": 2.6587, + "theoretical_loss": 3.494065786190328, + "tokens_seen": 1611530240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025847542627883654, + "loss": 2.7119, + "theoretical_loss": 3.4940534299560597, + "tokens_seen": 1611595776 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025846539618856566, + "loss": 2.5316, + "theoretical_loss": 3.4940410743649357, + "tokens_seen": 1611661312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584553660982949, + "loss": 2.4298, + "theoretical_loss": 3.494028719416897, + "tokens_seen": 1611726848 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584453360080241, + "loss": 2.6352, + "theoretical_loss": 3.4940163651118836, + "tokens_seen": 1611792384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025843530591775326, + "loss": 2.6358, + "theoretical_loss": 3.494004011449836, + "tokens_seen": 1611857920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025842527582748244, + "loss": 2.6609, + "theoretical_loss": 3.4939916584306943, + "tokens_seen": 1611923456 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584152457372116, + "loss": 2.6514, + "theoretical_loss": 3.493979306054399, + "tokens_seen": 1611988992 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584052156469408, + "loss": 2.7571, + "theoretical_loss": 3.493966954320891, + "tokens_seen": 1612054528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025839518555667004, + "loss": 2.5126, + "theoretical_loss": 3.49395460323011, + "tokens_seen": 1612120064 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 890307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.018338203430176, + "objective/train/theoretical_loss": 3.4939422527819968, + "objective/train/tokens_used": 1632645600, + "theoretical_loss": 3.4939422527819968, + "tokens_seen": 1612185600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025838515546639917, + "loss": 2.5821, + "theoretical_loss": 3.4939422527819968, + "tokens_seen": 1612185600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583751253761284, + "loss": 2.5408, + "theoretical_loss": 3.4939299029764923, + "tokens_seen": 1612251136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025836509528585753, + "loss": 2.4805, + "theoretical_loss": 3.493917553813536, + "tokens_seen": 1612316672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025835506519558677, + "loss": 2.6354, + "theoretical_loss": 3.493905205293069, + "tokens_seen": 1612382208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025834503510531595, + "loss": 2.7238, + "theoretical_loss": 3.4938928574150316, + "tokens_seen": 1612447744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025833500501504513, + "loss": 2.7054, + "theoretical_loss": 3.4938805101793644, + "tokens_seen": 1612513280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583249749247743, + "loss": 2.5066, + "theoretical_loss": 3.4938681635860074, + "tokens_seen": 1612578816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025831494483450354, + "loss": 2.7947, + "theoretical_loss": 3.4938558176349015, + "tokens_seen": 1612644352 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025830491474423267, + "loss": 2.5333, + "theoretical_loss": 3.4938434723259872, + "tokens_seen": 1612709888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582948846539619, + "loss": 2.6675, + "theoretical_loss": 3.4938311276592047, + "tokens_seen": 1612775424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025828485456369103, + "loss": 2.6112, + "theoretical_loss": 3.493818783634495, + "tokens_seen": 1612840960 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025827482447342027, + "loss": 2.6136, + "theoretical_loss": 3.493806440251798, + "tokens_seen": 1612906496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025826479438314945, + "loss": 2.6806, + "theoretical_loss": 3.4937940975110546, + "tokens_seen": 1612972032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025825476429287863, + "loss": 2.7465, + "theoretical_loss": 3.493781755412206, + "tokens_seen": 1613037568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025824473420260787, + "loss": 2.7219, + "theoretical_loss": 3.4937694139551914, + "tokens_seen": 1613103104 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258234704112337, + "loss": 2.5369, + "theoretical_loss": 3.4937570731399523, + "tokens_seen": 1613168640 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025822467402206623, + "loss": 2.7823, + "theoretical_loss": 3.4937447329664284, + "tokens_seen": 1613234176 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582146439317954, + "loss": 2.4617, + "theoretical_loss": 3.493732393434561, + "tokens_seen": 1613299712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582046138415246, + "loss": 2.6741, + "theoretical_loss": 3.4937200545442906, + "tokens_seen": 1613365248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025819458375125377, + "loss": 2.8025, + "theoretical_loss": 3.4937077162955577, + "tokens_seen": 1613430784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025818455366098295, + "loss": 2.7791, + "theoretical_loss": 3.4936953786883027, + "tokens_seen": 1613496320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025817452357071213, + "loss": 2.6624, + "theoretical_loss": 3.4936830417224662, + "tokens_seen": 1613561856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025816449348044137, + "loss": 2.5387, + "theoretical_loss": 3.4936707053979896, + "tokens_seen": 1613627392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581544633901705, + "loss": 2.5352, + "theoretical_loss": 3.4936583697148125, + "tokens_seen": 1613692928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025814443329989973, + "loss": 2.6936, + "theoretical_loss": 3.4936460346728753, + "tokens_seen": 1613758464 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 891421, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5894956588745117, + "objective/train/theoretical_loss": 3.49363370027212, + "objective/train/tokens_used": 1634284000, + "theoretical_loss": 3.49363370027212, + "tokens_seen": 1613824000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581344032096289, + "loss": 2.5598, + "theoretical_loss": 3.49363370027212, + "tokens_seen": 1613824000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581243731193581, + "loss": 2.6864, + "theoretical_loss": 3.493621366512486, + "tokens_seen": 1613889536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581143430290873, + "loss": 2.7065, + "theoretical_loss": 3.4936090333939145, + "tokens_seen": 1613955072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025810431293881646, + "loss": 2.512, + "theoretical_loss": 3.493596700916346, + "tokens_seen": 1614020608 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025809428284854564, + "loss": 2.579, + "theoretical_loss": 3.493584369079721, + "tokens_seen": 1614086144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025808425275827487, + "loss": 2.4163, + "theoretical_loss": 3.4935720378839807, + "tokens_seen": 1614151680 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258074222668004, + "loss": 2.4502, + "theoretical_loss": 3.493559707329065, + "tokens_seen": 1614217216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025806419257773323, + "loss": 2.7627, + "theoretical_loss": 3.4935473774149157, + "tokens_seen": 1614282752 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025805416248746236, + "loss": 2.5223, + "theoretical_loss": 3.4935350481414718, + "tokens_seen": 1614348288 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002580441323971916, + "loss": 2.5368, + "theoretical_loss": 3.4935227195086758, + "tokens_seen": 1614413824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002580341023069208, + "loss": 2.837, + "theoretical_loss": 3.4935103915164674, + "tokens_seen": 1614479360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025802407221664996, + "loss": 2.733, + "theoretical_loss": 3.4934980641647875, + "tokens_seen": 1614544896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025801404212637914, + "loss": 2.6878, + "theoretical_loss": 3.493485737453577, + "tokens_seen": 1614610432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002580040120361084, + "loss": 2.6848, + "theoretical_loss": 3.493473411382776, + "tokens_seen": 1614675968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579939819458375, + "loss": 2.3273, + "theoretical_loss": 3.493461085952326, + "tokens_seen": 1614741504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025798395185556674, + "loss": 2.4823, + "theoretical_loss": 3.493448761162168, + "tokens_seen": 1614807040 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025797392176529586, + "loss": 2.5212, + "theoretical_loss": 3.4934364370122415, + "tokens_seen": 1614872576 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579638916750251, + "loss": 2.8075, + "theoretical_loss": 3.493424113502488, + "tokens_seen": 1614938112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579538615847543, + "loss": 2.8885, + "theoretical_loss": 3.493411790632848, + "tokens_seen": 1615003648 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025794383149448346, + "loss": 2.8604, + "theoretical_loss": 3.493399468403263, + "tokens_seen": 1615069184 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025793380140421264, + "loss": 2.6701, + "theoretical_loss": 3.4933871468136735, + "tokens_seen": 1615134720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579237713139418, + "loss": 2.5409, + "theoretical_loss": 3.49337482586402, + "tokens_seen": 1615200256 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257913741223671, + "loss": 2.4081, + "theoretical_loss": 3.493362505554243, + "tokens_seen": 1615265792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025790371113340024, + "loss": 2.7342, + "theoretical_loss": 3.4933501858842844, + "tokens_seen": 1615331328 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025789368104312937, + "loss": 2.7004, + "theoretical_loss": 3.4933378668540835, + "tokens_seen": 1615396864 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 892490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.780491828918457, + "objective/train/theoretical_loss": 3.4933255484635826, + "objective/train/tokens_used": 1635922400, + "theoretical_loss": 3.4933255484635826, + "tokens_seen": 1615462400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578836509528586, + "loss": 2.6395, + "theoretical_loss": 3.4933255484635826, + "tokens_seen": 1615462400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025787362086258773, + "loss": 2.589, + "theoretical_loss": 3.4933132307127215, + "tokens_seen": 1615527936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025786359077231697, + "loss": 2.9653, + "theoretical_loss": 3.4933009136014412, + "tokens_seen": 1615593472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025785356068204615, + "loss": 2.6433, + "theoretical_loss": 3.493288597129683, + "tokens_seen": 1615659008 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025784353059177533, + "loss": 2.7118, + "theoretical_loss": 3.493276281297388, + "tokens_seen": 1615724544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578335005015045, + "loss": 2.8061, + "theoretical_loss": 3.4932639661044957, + "tokens_seen": 1615790080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025782347041123374, + "loss": 2.6577, + "theoretical_loss": 3.4932516515509486, + "tokens_seen": 1615855616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025781344032096287, + "loss": 2.7861, + "theoretical_loss": 3.4932393376366866, + "tokens_seen": 1615921152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578034102306921, + "loss": 2.6685, + "theoretical_loss": 3.4932270243616506, + "tokens_seen": 1615986688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025779338014042123, + "loss": 2.7509, + "theoretical_loss": 3.4932147117257824, + "tokens_seen": 1616052224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025778335005015047, + "loss": 2.4067, + "theoretical_loss": 3.4932023997290216, + "tokens_seen": 1616117760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025777331995987965, + "loss": 2.5045, + "theoretical_loss": 3.4931900883713096, + "tokens_seen": 1616183296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025776328986960883, + "loss": 2.6771, + "theoretical_loss": 3.493177777652588, + "tokens_seen": 1616248832 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257753259779338, + "loss": 2.6147, + "theoretical_loss": 3.493165467572797, + "tokens_seen": 1616314368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577432296890672, + "loss": 2.6992, + "theoretical_loss": 3.4931531581318778, + "tokens_seen": 1616379904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577331995987964, + "loss": 2.7035, + "theoretical_loss": 3.4931408493297713, + "tokens_seen": 1616445440 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577231695085256, + "loss": 2.7856, + "theoretical_loss": 3.493128541166418, + "tokens_seen": 1616510976 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025771313941825474, + "loss": 2.6062, + "theoretical_loss": 3.4931162336417594, + "tokens_seen": 1616576512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025770310932798397, + "loss": 2.6779, + "theoretical_loss": 3.4931039267557367, + "tokens_seen": 1616642048 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576930792377131, + "loss": 2.7855, + "theoretical_loss": 3.4930916205082903, + "tokens_seen": 1616707584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025768304914744233, + "loss": 2.7018, + "theoretical_loss": 3.493079314899361, + "tokens_seen": 1616773120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576730190571715, + "loss": 2.5119, + "theoretical_loss": 3.4930670099288905, + "tokens_seen": 1616838656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576629889669007, + "loss": 2.6613, + "theoretical_loss": 3.49305470559682, + "tokens_seen": 1616904192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576529588766299, + "loss": 2.8252, + "theoretical_loss": 3.4930424019030895, + "tokens_seen": 1616969728 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576429287863591, + "loss": 2.7648, + "theoretical_loss": 3.4930300988476404, + "tokens_seen": 1617035264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 893217, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8498308658599854, + "objective/train/theoretical_loss": 3.493017796430414, + "objective/train/tokens_used": 1637560800, + "theoretical_loss": 3.493017796430414, + "tokens_seen": 1617100800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025763289869608824, + "loss": 2.6071, + "theoretical_loss": 3.493017796430414, + "tokens_seen": 1617100800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576228686058175, + "loss": 2.7201, + "theoretical_loss": 3.493005494651351, + "tokens_seen": 1617166336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576128385155466, + "loss": 2.7744, + "theoretical_loss": 3.492993193510393, + "tokens_seen": 1617231872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025760280842527584, + "loss": 2.7385, + "theoretical_loss": 3.49298089300748, + "tokens_seen": 1617297408 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257592778335005, + "loss": 2.7485, + "theoretical_loss": 3.492968593142554, + "tokens_seen": 1617362944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575827482447342, + "loss": 2.6081, + "theoretical_loss": 3.492956293915556, + "tokens_seen": 1617428480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575727181544634, + "loss": 2.6366, + "theoretical_loss": 3.4929439953264265, + "tokens_seen": 1617494016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025756268806419256, + "loss": 2.5578, + "theoretical_loss": 3.4929316973751066, + "tokens_seen": 1617559552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025755265797392174, + "loss": 2.6992, + "theoretical_loss": 3.4929194000615382, + "tokens_seen": 1617625088 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257542627883651, + "loss": 2.7894, + "theoretical_loss": 3.4929071033856616, + "tokens_seen": 1617690624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575325977933801, + "loss": 2.6224, + "theoretical_loss": 3.4928948073474184, + "tokens_seen": 1617756160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025752256770310934, + "loss": 2.8395, + "theoretical_loss": 3.4928825119467497, + "tokens_seen": 1617821696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025751253761283847, + "loss": 2.6206, + "theoretical_loss": 3.492870217183596, + "tokens_seen": 1617887232 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575025075225677, + "loss": 2.71, + "theoretical_loss": 3.4928579230578984, + "tokens_seen": 1617952768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025749247743229694, + "loss": 2.5746, + "theoretical_loss": 3.492845629569599, + "tokens_seen": 1618018304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025748244734202606, + "loss": 2.5882, + "theoretical_loss": 3.4928333367186384, + "tokens_seen": 1618083840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574724172517553, + "loss": 2.5893, + "theoretical_loss": 3.4928210445049577, + "tokens_seen": 1618149376 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574623871614845, + "loss": 2.8691, + "theoretical_loss": 3.492808752928498, + "tokens_seen": 1618214912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025745235707121366, + "loss": 2.6229, + "theoretical_loss": 3.4927964619892005, + "tokens_seen": 1618280448 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025744232698094284, + "loss": 2.6035, + "theoretical_loss": 3.4927841716870063, + "tokens_seen": 1618345984 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257432296890672, + "loss": 2.7588, + "theoretical_loss": 3.4927718820218567, + "tokens_seen": 1618411520 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574222668004012, + "loss": 2.522, + "theoretical_loss": 3.4927595929936928, + "tokens_seen": 1618477056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025741223671013044, + "loss": 2.5641, + "theoretical_loss": 3.4927473046024566, + "tokens_seen": 1618542592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025740220661985957, + "loss": 2.5197, + "theoretical_loss": 3.492735016848088, + "tokens_seen": 1618608128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573921765295888, + "loss": 2.4852, + "theoretical_loss": 3.4927227297305286, + "tokens_seen": 1618673664 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 894105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0400853157043457, + "objective/train/theoretical_loss": 3.4927104432497202, + "objective/train/tokens_used": 1639199200, + "theoretical_loss": 3.4927104432497202, + "tokens_seen": 1618739200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025738214643931793, + "loss": 2.7416, + "theoretical_loss": 3.4927104432497202, + "tokens_seen": 1618739200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025737211634904717, + "loss": 2.8181, + "theoretical_loss": 3.492698157405603, + "tokens_seen": 1618804736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025736208625877635, + "loss": 2.7422, + "theoretical_loss": 3.4926858721981198, + "tokens_seen": 1618870272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025735205616850553, + "loss": 2.5819, + "theoretical_loss": 3.49267358762721, + "tokens_seen": 1618935808 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573420260782347, + "loss": 2.78, + "theoretical_loss": 3.492661303692816, + "tokens_seen": 1619001344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025733199598796394, + "loss": 2.7969, + "theoretical_loss": 3.492649020394879, + "tokens_seen": 1619066880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025732196589769307, + "loss": 2.5117, + "theoretical_loss": 3.4926367377333403, + "tokens_seen": 1619132416 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573119358074223, + "loss": 2.7497, + "theoretical_loss": 3.4926244557081403, + "tokens_seen": 1619197952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025730190571715143, + "loss": 2.3726, + "theoretical_loss": 3.492612174319221, + "tokens_seen": 1619263488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025729187562688067, + "loss": 2.4969, + "theoretical_loss": 3.492599893566524, + "tokens_seen": 1619329024 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025728184553660985, + "loss": 2.7764, + "theoretical_loss": 3.49258761344999, + "tokens_seen": 1619394560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025727181544633903, + "loss": 2.7566, + "theoretical_loss": 3.4925753339695604, + "tokens_seen": 1619460096 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572617853560682, + "loss": 2.5046, + "theoretical_loss": 3.492563055125177, + "tokens_seen": 1619525632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572517552657974, + "loss": 2.8057, + "theoretical_loss": 3.4925507769167803, + "tokens_seen": 1619591168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572417251755266, + "loss": 2.6301, + "theoretical_loss": 3.4925384993443123, + "tokens_seen": 1619656704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572316950852558, + "loss": 2.5833, + "theoretical_loss": 3.492526222407714, + "tokens_seen": 1619722240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025722166499498494, + "loss": 2.6346, + "theoretical_loss": 3.492513946106927, + "tokens_seen": 1619787776 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025721163490471417, + "loss": 2.6943, + "theoretical_loss": 3.4925016704418916, + "tokens_seen": 1619853312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572016048144433, + "loss": 2.5891, + "theoretical_loss": 3.4924893954125507, + "tokens_seen": 1619918848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025719157472417253, + "loss": 2.5932, + "theoretical_loss": 3.492477121018845, + "tokens_seen": 1619984384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571815446339017, + "loss": 2.7712, + "theoretical_loss": 3.492464847260716, + "tokens_seen": 1620049920 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571715145436309, + "loss": 2.9281, + "theoretical_loss": 3.4924525741381047, + "tokens_seen": 1620115456 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571614844533601, + "loss": 2.7198, + "theoretical_loss": 3.492440301650953, + "tokens_seen": 1620180992 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571514543630893, + "loss": 2.5301, + "theoretical_loss": 3.4924280297992016, + "tokens_seen": 1620246528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025714142427281844, + "loss": 2.5522, + "theoretical_loss": 3.4924157585827924, + "tokens_seen": 1620312064 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 894566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7740554809570312, + "objective/train/theoretical_loss": 3.492403488001667, + "objective/train/tokens_used": 1640837600, + "theoretical_loss": 3.492403488001667, + "tokens_seen": 1620377600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571313941825477, + "loss": 2.6027, + "theoretical_loss": 3.492403488001667, + "tokens_seen": 1620377600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571213640922768, + "loss": 2.4531, + "theoretical_loss": 3.492391218055767, + "tokens_seen": 1620443136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025711133400200604, + "loss": 2.5117, + "theoretical_loss": 3.4923789487450327, + "tokens_seen": 1620508672 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571013039117352, + "loss": 2.5483, + "theoretical_loss": 3.492366680069406, + "tokens_seen": 1620574208 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570912738214644, + "loss": 2.5429, + "theoretical_loss": 3.4923544120288295, + "tokens_seen": 1620639744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570812437311936, + "loss": 2.9547, + "theoretical_loss": 3.492342144623243, + "tokens_seen": 1620705280 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025707121364092276, + "loss": 2.407, + "theoretical_loss": 3.492329877852589, + "tokens_seen": 1620770816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025706118355065194, + "loss": 2.6359, + "theoretical_loss": 3.4923176117168087, + "tokens_seen": 1620836352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570511534603812, + "loss": 2.7424, + "theoretical_loss": 3.492305346215843, + "tokens_seen": 1620901888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570411233701103, + "loss": 2.5256, + "theoretical_loss": 3.4922930813496347, + "tokens_seen": 1620967424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025703109327983954, + "loss": 2.8147, + "theoretical_loss": 3.492280817118124, + "tokens_seen": 1621032960 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025702106318956867, + "loss": 2.7526, + "theoretical_loss": 3.492268553521253, + "tokens_seen": 1621098496 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570110330992979, + "loss": 2.453, + "theoretical_loss": 3.492256290558963, + "tokens_seen": 1621164032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570010030090271, + "loss": 2.7507, + "theoretical_loss": 3.4922440282311955, + "tokens_seen": 1621229568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025699097291875626, + "loss": 2.6752, + "theoretical_loss": 3.492231766537892, + "tokens_seen": 1621295104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025698094282848545, + "loss": 2.7155, + "theoretical_loss": 3.4922195054789946, + "tokens_seen": 1621360640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569709127382147, + "loss": 2.6698, + "theoretical_loss": 3.492207245054444, + "tokens_seen": 1621426176 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569608826479438, + "loss": 2.4975, + "theoretical_loss": 3.4921949852641827, + "tokens_seen": 1621491712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025695085255767304, + "loss": 2.4095, + "theoretical_loss": 3.492182726108151, + "tokens_seen": 1621557248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025694082246740217, + "loss": 2.6184, + "theoretical_loss": 3.4921704675862912, + "tokens_seen": 1621622784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569307923771314, + "loss": 2.6408, + "theoretical_loss": 3.4921582096985455, + "tokens_seen": 1621688320 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569207622868606, + "loss": 2.4466, + "theoretical_loss": 3.4921459524448544, + "tokens_seen": 1621753856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025691073219658977, + "loss": 2.6971, + "theoretical_loss": 3.4921336958251596, + "tokens_seen": 1621819392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025690070210631895, + "loss": 2.5353, + "theoretical_loss": 3.492121439839403, + "tokens_seen": 1621884928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025689067201604813, + "loss": 2.5513, + "theoretical_loss": 3.4921091844875267, + "tokens_seen": 1621950464 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 895662, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5183286666870117, + "objective/train/theoretical_loss": 3.4920969297694713, + "objective/train/tokens_used": 1642476000, + "theoretical_loss": 3.4920969297694713, + "tokens_seen": 1622016000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568806419257773, + "loss": 2.7824, + "theoretical_loss": 3.4920969297694713, + "tokens_seen": 1622016000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025687061183550655, + "loss": 2.6162, + "theoretical_loss": 3.4920846756851787, + "tokens_seen": 1622081536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568605817452357, + "loss": 2.7633, + "theoretical_loss": 3.4920724222345907, + "tokens_seen": 1622147072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568505516549649, + "loss": 2.6291, + "theoretical_loss": 3.492060169417649, + "tokens_seen": 1622212608 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025684052156469404, + "loss": 2.5131, + "theoretical_loss": 3.492047917234296, + "tokens_seen": 1622278144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025683049147442327, + "loss": 2.6453, + "theoretical_loss": 3.4920356656844715, + "tokens_seen": 1622343680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025682046138415245, + "loss": 2.597, + "theoretical_loss": 3.492023414768118, + "tokens_seen": 1622409216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025681043129388163, + "loss": 2.6343, + "theoretical_loss": 3.492011164485178, + "tokens_seen": 1622474752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568004012036108, + "loss": 2.5363, + "theoretical_loss": 3.4919989148355928, + "tokens_seen": 1622540288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025679037111334005, + "loss": 2.5657, + "theoretical_loss": 3.4919866658193026, + "tokens_seen": 1622605824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567803410230692, + "loss": 2.6233, + "theoretical_loss": 3.491974417436251, + "tokens_seen": 1622671360 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567703109327984, + "loss": 2.6616, + "theoretical_loss": 3.4919621696863787, + "tokens_seen": 1622736896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025676028084252754, + "loss": 2.3667, + "theoretical_loss": 3.4919499225696278, + "tokens_seen": 1622802432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567502507522568, + "loss": 2.6855, + "theoretical_loss": 3.4919376760859397, + "tokens_seen": 1622867968 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256740220661986, + "loss": 2.9121, + "theoretical_loss": 3.4919254302352565, + "tokens_seen": 1622933504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025673019057171514, + "loss": 2.7713, + "theoretical_loss": 3.491913185017519, + "tokens_seen": 1622999040 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025672016048144437, + "loss": 2.4044, + "theoretical_loss": 3.49190094043267, + "tokens_seen": 1623064576 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567101303911735, + "loss": 2.543, + "theoretical_loss": 3.4918886964806513, + "tokens_seen": 1623130112 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025670010030090273, + "loss": 2.8006, + "theoretical_loss": 3.4918764531614035, + "tokens_seen": 1623195648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566900702106319, + "loss": 2.7173, + "theoretical_loss": 3.491864210474869, + "tokens_seen": 1623261184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566800401203611, + "loss": 2.717, + "theoretical_loss": 3.49185196842099, + "tokens_seen": 1623326720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566700100300903, + "loss": 2.4206, + "theoretical_loss": 3.4918397269997077, + "tokens_seen": 1623392256 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566599799398195, + "loss": 2.7348, + "theoretical_loss": 3.491827486210964, + "tokens_seen": 1623457792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025664994984954864, + "loss": 2.5949, + "theoretical_loss": 3.491815246054701, + "tokens_seen": 1623523328 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566399197592779, + "loss": 2.8221, + "theoretical_loss": 3.49180300653086, + "tokens_seen": 1623588864 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 896620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.650221347808838, + "objective/train/theoretical_loss": 3.4917907676393827, + "objective/train/tokens_used": 1644114400, + "theoretical_loss": 3.4917907676393827, + "tokens_seen": 1623654400 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256629889669007, + "loss": 2.4727, + "theoretical_loss": 3.4917907676393827, + "tokens_seen": 1623654400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025661985957873624, + "loss": 2.7314, + "theoretical_loss": 3.4917785293802117, + "tokens_seen": 1623719936 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566098294884654, + "loss": 2.6896, + "theoretical_loss": 3.4917662917532875, + "tokens_seen": 1623785472 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565997993981946, + "loss": 2.7967, + "theoretical_loss": 3.4917540547585535, + "tokens_seen": 1623851008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565897693079238, + "loss": 2.4523, + "theoretical_loss": 3.4917418183959508, + "tokens_seen": 1623916544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025657973921765296, + "loss": 2.4015, + "theoretical_loss": 3.491729582665421, + "tokens_seen": 1623982080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025656970912738214, + "loss": 2.8635, + "theoretical_loss": 3.4917173475669063, + "tokens_seen": 1624047616 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565596790371114, + "loss": 2.638, + "theoretical_loss": 3.4917051131003483, + "tokens_seen": 1624113152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565496489468405, + "loss": 2.6574, + "theoretical_loss": 3.4916928792656887, + "tokens_seen": 1624178688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025653961885656974, + "loss": 2.6169, + "theoretical_loss": 3.49168064606287, + "tokens_seen": 1624244224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025652958876629887, + "loss": 2.6505, + "theoretical_loss": 3.4916684134918334, + "tokens_seen": 1624309760 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565195586760281, + "loss": 2.8218, + "theoretical_loss": 3.4916561815525213, + "tokens_seen": 1624375296 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565095285857573, + "loss": 2.7438, + "theoretical_loss": 3.4916439502448755, + "tokens_seen": 1624440832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025649949849548647, + "loss": 2.5903, + "theoretical_loss": 3.4916317195688373, + "tokens_seen": 1624506368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025648946840521565, + "loss": 2.6318, + "theoretical_loss": 3.49161948952435, + "tokens_seen": 1624571904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564794383149449, + "loss": 2.4337, + "theoretical_loss": 3.491607260111354, + "tokens_seen": 1624637440 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256469408224674, + "loss": 2.681, + "theoretical_loss": 3.4915950313297914, + "tokens_seen": 1624702976 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025645937813440324, + "loss": 2.821, + "theoretical_loss": 3.4915828031796052, + "tokens_seen": 1624768512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025644934804413237, + "loss": 2.5816, + "theoretical_loss": 3.4915705756607363, + "tokens_seen": 1624834048 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564393179538616, + "loss": 2.5739, + "theoretical_loss": 3.491558348773127, + "tokens_seen": 1624899584 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564292878635908, + "loss": 2.5323, + "theoretical_loss": 3.49154612251672, + "tokens_seen": 1624965120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025641925777331997, + "loss": 2.7479, + "theoretical_loss": 3.4915338968914558, + "tokens_seen": 1625030656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025640922768304915, + "loss": 2.6312, + "theoretical_loss": 3.491521671897278, + "tokens_seen": 1625096192 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025639919759277833, + "loss": 2.6383, + "theoretical_loss": 3.491509447534127, + "tokens_seen": 1625161728 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563891675025075, + "loss": 2.5722, + "theoretical_loss": 3.4914972238019453, + "tokens_seen": 1625227264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 896956, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6397101879119873, + "objective/train/theoretical_loss": 3.491485000700675, + "objective/train/tokens_used": 1645752800, + "theoretical_loss": 3.491485000700675, + "tokens_seen": 1625292800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025637913741223675, + "loss": 2.5791, + "theoretical_loss": 3.491485000700675, + "tokens_seen": 1625292800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563691073219659, + "loss": 2.6691, + "theoretical_loss": 3.491472778230259, + "tokens_seen": 1625358336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563590772316951, + "loss": 2.6711, + "theoretical_loss": 3.4914605563906376, + "tokens_seen": 1625423872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025634904714142424, + "loss": 2.8036, + "theoretical_loss": 3.4914483351817545, + "tokens_seen": 1625489408 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025633901705115347, + "loss": 2.6206, + "theoretical_loss": 3.49143611460355, + "tokens_seen": 1625554944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025632898696088265, + "loss": 2.7595, + "theoretical_loss": 3.4914238946559673, + "tokens_seen": 1625620480 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025631895687061183, + "loss": 2.5415, + "theoretical_loss": 3.4914116753389486, + "tokens_seen": 1625686016 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256308926780341, + "loss": 2.663, + "theoretical_loss": 3.4913994566524353, + "tokens_seen": 1625751552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025629889669007025, + "loss": 2.8635, + "theoretical_loss": 3.49138723859637, + "tokens_seen": 1625817088 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562888665997994, + "loss": 2.7911, + "theoretical_loss": 3.4913750211706938, + "tokens_seen": 1625882624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562788365095286, + "loss": 2.7812, + "theoretical_loss": 3.4913628043753495, + "tokens_seen": 1625948160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025626880641925774, + "loss": 2.7115, + "theoretical_loss": 3.4913505882102793, + "tokens_seen": 1626013696 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256258776328987, + "loss": 2.7131, + "theoretical_loss": 3.491338372675425, + "tokens_seen": 1626079232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025624874623871616, + "loss": 2.4298, + "theoretical_loss": 3.491326157770729, + "tokens_seen": 1626144768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025623871614844534, + "loss": 2.5261, + "theoretical_loss": 3.4913139434961327, + "tokens_seen": 1626210304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562286860581745, + "loss": 2.7258, + "theoretical_loss": 3.4913017298515783, + "tokens_seen": 1626275840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562186559679037, + "loss": 2.822, + "theoretical_loss": 3.491289516837009, + "tokens_seen": 1626341376 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562086258776329, + "loss": 2.6712, + "theoretical_loss": 3.4912773044523657, + "tokens_seen": 1626406912 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561985957873621, + "loss": 2.7299, + "theoretical_loss": 3.491265092697591, + "tokens_seen": 1626472448 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025618856569709124, + "loss": 2.5132, + "theoretical_loss": 3.4912528815726276, + "tokens_seen": 1626537984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561785356068205, + "loss": 2.7295, + "theoretical_loss": 3.491240671077417, + "tokens_seen": 1626603520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025616850551654966, + "loss": 2.7413, + "theoretical_loss": 3.491228461211901, + "tokens_seen": 1626669056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025615847542627884, + "loss": 2.7521, + "theoretical_loss": 3.4912162519760224, + "tokens_seen": 1626734592 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256148445336008, + "loss": 2.6282, + "theoretical_loss": 3.4912040433697236, + "tokens_seen": 1626800128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561384152457372, + "loss": 2.5982, + "theoretical_loss": 3.4911918353929456, + "tokens_seen": 1626865664 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 898227, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.830223321914673, + "objective/train/theoretical_loss": 3.4911796280456313, + "objective/train/tokens_used": 1647391200, + "theoretical_loss": 3.4911796280456313, + "tokens_seen": 1626931200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561283851554664, + "loss": 2.7122, + "theoretical_loss": 3.4911796280456313, + "tokens_seen": 1626931200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561183550651956, + "loss": 2.7158, + "theoretical_loss": 3.4911674213277237, + "tokens_seen": 1626996736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025610832497492475, + "loss": 2.6925, + "theoretical_loss": 3.4911552152391634, + "tokens_seen": 1627062272 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256098294884654, + "loss": 2.5707, + "theoretical_loss": 3.491143009779894, + "tokens_seen": 1627127808 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560882647943831, + "loss": 2.9297, + "theoretical_loss": 3.491130804949857, + "tokens_seen": 1627193344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025607823470411234, + "loss": 2.7407, + "theoretical_loss": 3.4911186007489947, + "tokens_seen": 1627258880 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560682046138415, + "loss": 2.6176, + "theoretical_loss": 3.491106397177249, + "tokens_seen": 1627324416 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560581745235707, + "loss": 2.6364, + "theoretical_loss": 3.491094194234563, + "tokens_seen": 1627389952 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560481444332999, + "loss": 2.7198, + "theoretical_loss": 3.491081991920878, + "tokens_seen": 1627455488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025603811434302907, + "loss": 2.7497, + "theoretical_loss": 3.4910697902361374, + "tokens_seen": 1627521024 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025602808425275825, + "loss": 2.499, + "theoretical_loss": 3.491057589180282, + "tokens_seen": 1627586560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560180541624875, + "loss": 2.5999, + "theoretical_loss": 3.491045388753255, + "tokens_seen": 1627652096 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560080240722166, + "loss": 2.6363, + "theoretical_loss": 3.491033188954999, + "tokens_seen": 1627717632 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025599799398194585, + "loss": 2.731, + "theoretical_loss": 3.491020989785455, + "tokens_seen": 1627783168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559879638916751, + "loss": 2.7878, + "theoretical_loss": 3.4910087912445666, + "tokens_seen": 1627848704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559779338014042, + "loss": 2.7669, + "theoretical_loss": 3.4909965933322757, + "tokens_seen": 1627914240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025596790371113344, + "loss": 2.5268, + "theoretical_loss": 3.4909843960485243, + "tokens_seen": 1627979776 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025595787362086257, + "loss": 2.6021, + "theoretical_loss": 3.490972199393255, + "tokens_seen": 1628045312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559478435305918, + "loss": 2.6335, + "theoretical_loss": 3.4909600033664097, + "tokens_seen": 1628110848 + }, + { + "epoch": 0.49, + "learning_rate": 0.000255937813440321, + "loss": 2.4764, + "theoretical_loss": 3.490947807967931, + "tokens_seen": 1628176384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025592778335005017, + "loss": 2.8449, + "theoretical_loss": 3.4909356131977614, + "tokens_seen": 1628241920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025591775325977935, + "loss": 2.7262, + "theoretical_loss": 3.4909234190558434, + "tokens_seen": 1628307456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025590772316950853, + "loss": 2.5077, + "theoretical_loss": 3.490911225542119, + "tokens_seen": 1628372992 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558976930792377, + "loss": 2.6785, + "theoretical_loss": 3.4908990326565306, + "tokens_seen": 1628438528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025588766298896695, + "loss": 2.7593, + "theoretical_loss": 3.4908868403990203, + "tokens_seen": 1628504064 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 898909, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.812804937362671, + "objective/train/theoretical_loss": 3.4908746487695304, + "objective/train/tokens_used": 1649029600, + "theoretical_loss": 3.4908746487695304, + "tokens_seen": 1628569600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558776328986961, + "loss": 2.7889, + "theoretical_loss": 3.4908746487695304, + "tokens_seen": 1628569600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558676028084253, + "loss": 2.5556, + "theoretical_loss": 3.4908624577680043, + "tokens_seen": 1628635136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025585757271815444, + "loss": 2.4617, + "theoretical_loss": 3.4908502673943835, + "tokens_seen": 1628700672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025584754262788367, + "loss": 2.6531, + "theoretical_loss": 3.4908380776486108, + "tokens_seen": 1628766208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025583751253761285, + "loss": 2.7881, + "theoretical_loss": 3.4908258885306283, + "tokens_seen": 1628831744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025582748244734203, + "loss": 2.8264, + "theoretical_loss": 3.4908137000403783, + "tokens_seen": 1628897280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558174523570712, + "loss": 2.5923, + "theoretical_loss": 3.4908015121778035, + "tokens_seen": 1628962816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025580742226680045, + "loss": 2.6108, + "theoretical_loss": 3.4907893249428463, + "tokens_seen": 1629028352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557973921765296, + "loss": 2.5873, + "theoretical_loss": 3.4907771383354493, + "tokens_seen": 1629093888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557873620862588, + "loss": 2.4197, + "theoretical_loss": 3.490764952355555, + "tokens_seen": 1629159424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025577733199598794, + "loss": 2.9189, + "theoretical_loss": 3.490752767003105, + "tokens_seen": 1629224960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557673019057172, + "loss": 2.6294, + "theoretical_loss": 3.4907405822780424, + "tokens_seen": 1629290496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025575727181544636, + "loss": 2.7208, + "theoretical_loss": 3.49072839818031, + "tokens_seen": 1629356032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025574724172517554, + "loss": 2.5332, + "theoretical_loss": 3.49071621470985, + "tokens_seen": 1629421568 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557372116349047, + "loss": 2.7087, + "theoretical_loss": 3.4907040318666045, + "tokens_seen": 1629487104 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557271815446339, + "loss": 2.4777, + "theoretical_loss": 3.490691849650516, + "tokens_seen": 1629552640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557171514543631, + "loss": 2.6762, + "theoretical_loss": 3.4906796680615275, + "tokens_seen": 1629618176 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557071213640923, + "loss": 2.6705, + "theoretical_loss": 3.4906674870995813, + "tokens_seen": 1629683712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025569709127382144, + "loss": 2.6812, + "theoretical_loss": 3.4906553067646198, + "tokens_seen": 1629749248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556870611835507, + "loss": 2.6065, + "theoretical_loss": 3.490643127056585, + "tokens_seen": 1629814784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025567703109327986, + "loss": 2.8383, + "theoretical_loss": 3.490630947975421, + "tokens_seen": 1629880320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025566700100300904, + "loss": 2.4477, + "theoretical_loss": 3.490618769521069, + "tokens_seen": 1629945856 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556569709127382, + "loss": 2.6374, + "theoretical_loss": 3.4906065916934716, + "tokens_seen": 1630011392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556469408224674, + "loss": 2.767, + "theoretical_loss": 3.490594414492572, + "tokens_seen": 1630076928 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556369107321966, + "loss": 2.6867, + "theoretical_loss": 3.4905822379183125, + "tokens_seen": 1630142464 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 899823, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1974503993988037, + "objective/train/theoretical_loss": 3.490570061970635, + "objective/train/tokens_used": 1650668000, + "theoretical_loss": 3.490570061970635, + "tokens_seen": 1630208000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556268806419258, + "loss": 2.8756, + "theoretical_loss": 3.490570061970635, + "tokens_seen": 1630208000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025561685055165495, + "loss": 2.7593, + "theoretical_loss": 3.4905578866494826, + "tokens_seen": 1630273536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556068204613842, + "loss": 2.5659, + "theoretical_loss": 3.4905457119547982, + "tokens_seen": 1630339072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555967903711133, + "loss": 2.6666, + "theoretical_loss": 3.490533537886524, + "tokens_seen": 1630404608 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025558676028084254, + "loss": 2.5991, + "theoretical_loss": 3.4905213644446027, + "tokens_seen": 1630470144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555767301905717, + "loss": 2.6245, + "theoretical_loss": 3.490509191628977, + "tokens_seen": 1630535680 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555667001003009, + "loss": 2.3732, + "theoretical_loss": 3.490497019439589, + "tokens_seen": 1630601216 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555566700100301, + "loss": 2.721, + "theoretical_loss": 3.490484847876382, + "tokens_seen": 1630666752 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025554663991975927, + "loss": 2.7833, + "theoretical_loss": 3.490472676939298, + "tokens_seen": 1630732288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025553660982948845, + "loss": 2.5654, + "theoretical_loss": 3.4904605066282803, + "tokens_seen": 1630797824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555265797392177, + "loss": 2.5818, + "theoretical_loss": 3.4904483369432704, + "tokens_seen": 1630863360 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555165496489468, + "loss": 2.7179, + "theoretical_loss": 3.4904361678842126, + "tokens_seen": 1630928896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025550651955867605, + "loss": 2.4192, + "theoretical_loss": 3.4904239994510484, + "tokens_seen": 1630994432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025549648946840523, + "loss": 2.4348, + "theoretical_loss": 3.490411831643721, + "tokens_seen": 1631059968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554864593781344, + "loss": 2.7202, + "theoretical_loss": 3.4903996644621724, + "tokens_seen": 1631125504 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554764292878636, + "loss": 2.6147, + "theoretical_loss": 3.4903874979063456, + "tokens_seen": 1631191040 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025546639919759277, + "loss": 2.438, + "theoretical_loss": 3.4903753319761837, + "tokens_seen": 1631256576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025545636910732195, + "loss": 2.8955, + "theoretical_loss": 3.490363166671629, + "tokens_seen": 1631322112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554463390170512, + "loss": 2.5263, + "theoretical_loss": 3.4903510019926243, + "tokens_seen": 1631387648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554363089267803, + "loss": 2.7364, + "theoretical_loss": 3.4903388379391123, + "tokens_seen": 1631453184 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025542627883650955, + "loss": 2.8961, + "theoretical_loss": 3.490326674511035, + "tokens_seen": 1631518720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554162487462387, + "loss": 2.5371, + "theoretical_loss": 3.4903145117083367, + "tokens_seen": 1631584256 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554062186559679, + "loss": 2.4477, + "theoretical_loss": 3.4903023495309586, + "tokens_seen": 1631649792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553961885656971, + "loss": 2.5275, + "theoretical_loss": 3.490290187978845, + "tokens_seen": 1631715328 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553861584754263, + "loss": 2.6323, + "theoretical_loss": 3.4902780270519367, + "tokens_seen": 1631780864 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 900688, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.873173475265503, + "objective/train/theoretical_loss": 3.4902658667501774, + "objective/train/tokens_used": 1652306400, + "theoretical_loss": 3.4902658667501774, + "tokens_seen": 1631846400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025537612838515546, + "loss": 2.6775, + "theoretical_loss": 3.4902658667501774, + "tokens_seen": 1631846400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025536609829488464, + "loss": 2.518, + "theoretical_loss": 3.4902537070735105, + "tokens_seen": 1631911936 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553560682046138, + "loss": 2.7277, + "theoretical_loss": 3.490241548021878, + "tokens_seen": 1631977472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025534603811434305, + "loss": 2.6138, + "theoretical_loss": 3.490229389595223, + "tokens_seen": 1632043008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553360080240722, + "loss": 2.5422, + "theoretical_loss": 3.490217231793488, + "tokens_seen": 1632108544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553259779338014, + "loss": 2.5416, + "theoretical_loss": 3.4902050746166156, + "tokens_seen": 1632174080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553159478435306, + "loss": 2.4946, + "theoretical_loss": 3.4901929180645492, + "tokens_seen": 1632239616 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553059177532598, + "loss": 2.7374, + "theoretical_loss": 3.490180762137231, + "tokens_seen": 1632305152 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025529588766298896, + "loss": 2.3845, + "theoretical_loss": 3.4901686068346045, + "tokens_seen": 1632370688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025528585757271814, + "loss": 2.7952, + "theoretical_loss": 3.4901564521566124, + "tokens_seen": 1632436224 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552758274824473, + "loss": 2.6978, + "theoretical_loss": 3.490144298103197, + "tokens_seen": 1632501760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025526579739217656, + "loss": 2.5157, + "theoretical_loss": 3.4901321446743014, + "tokens_seen": 1632567296 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552557673019057, + "loss": 2.7612, + "theoretical_loss": 3.490119991869869, + "tokens_seen": 1632632832 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552457372116349, + "loss": 2.7876, + "theoretical_loss": 3.4901078396898413, + "tokens_seen": 1632698368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552357071213641, + "loss": 2.6199, + "theoretical_loss": 3.4900956881341623, + "tokens_seen": 1632763904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552256770310933, + "loss": 2.5505, + "theoretical_loss": 3.490083537202774, + "tokens_seen": 1632829440 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552156469408225, + "loss": 2.7916, + "theoretical_loss": 3.490071386895621, + "tokens_seen": 1632894976 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025520561685055164, + "loss": 2.4555, + "theoretical_loss": 3.4900592372126438, + "tokens_seen": 1632960512 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551955867602809, + "loss": 2.7613, + "theoretical_loss": 3.490047088153787, + "tokens_seen": 1633026048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025518555667001006, + "loss": 2.2399, + "theoretical_loss": 3.490034939718993, + "tokens_seen": 1633091584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025517552657973924, + "loss": 2.697, + "theoretical_loss": 3.4900227919082045, + "tokens_seen": 1633157120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551654964894684, + "loss": 2.6479, + "theoretical_loss": 3.490010644721365, + "tokens_seen": 1633222656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551554663991976, + "loss": 2.7111, + "theoretical_loss": 3.4899984981584167, + "tokens_seen": 1633288192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551454363089268, + "loss": 2.6725, + "theoretical_loss": 3.4899863522193026, + "tokens_seen": 1633353728 + }, + { + "epoch": 0.49, + "learning_rate": 0.000255135406218656, + "loss": 2.6307, + "theoretical_loss": 3.489974206903966, + "tokens_seen": 1633419264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 901970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6137895584106445, + "objective/train/theoretical_loss": 3.4899620622123497, + "objective/train/tokens_used": 1653944800, + "theoretical_loss": 3.4899620622123497, + "tokens_seen": 1633484800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025512537612838515, + "loss": 2.3214, + "theoretical_loss": 3.4899620622123497, + "tokens_seen": 1633484800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002551153460381144, + "loss": 2.8042, + "theoretical_loss": 3.4899499181443967, + "tokens_seen": 1633550336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002551053159478435, + "loss": 2.7133, + "theoretical_loss": 3.4899377747000497, + "tokens_seen": 1633615872 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025509528585757274, + "loss": 2.7481, + "theoretical_loss": 3.489925631879252, + "tokens_seen": 1633681408 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550852557673019, + "loss": 2.6438, + "theoretical_loss": 3.489913489681946, + "tokens_seen": 1633746944 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550752256770311, + "loss": 2.6686, + "theoretical_loss": 3.4899013481080763, + "tokens_seen": 1633812480 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550651955867603, + "loss": 2.8043, + "theoretical_loss": 3.4898892071575833, + "tokens_seen": 1633878016 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025505516549648947, + "loss": 2.4947, + "theoretical_loss": 3.489877066830412, + "tokens_seen": 1633943552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025504513540621865, + "loss": 2.4075, + "theoretical_loss": 3.489864927126505, + "tokens_seen": 1634009088 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550351053159479, + "loss": 2.8582, + "theoretical_loss": 3.4898527880458046, + "tokens_seen": 1634074624 + }, + { + "epoch": 0.5, + "learning_rate": 0.000255025075225677, + "loss": 2.4703, + "theoretical_loss": 3.4898406495882544, + "tokens_seen": 1634140160 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025501504513540625, + "loss": 2.4688, + "theoretical_loss": 3.4898285117537977, + "tokens_seen": 1634205696 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025500501504513543, + "loss": 2.5317, + "theoretical_loss": 3.489816374542377, + "tokens_seen": 1634271232 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549949849548646, + "loss": 2.4912, + "theoretical_loss": 3.489804237953935, + "tokens_seen": 1634336768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549849548645938, + "loss": 2.5076, + "theoretical_loss": 3.489792101988416, + "tokens_seen": 1634402304 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025497492477432297, + "loss": 2.7753, + "theoretical_loss": 3.4897799666457616, + "tokens_seen": 1634467840 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025496489468405215, + "loss": 2.548, + "theoretical_loss": 3.489767831925916, + "tokens_seen": 1634533376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549548645937814, + "loss": 2.8576, + "theoretical_loss": 3.4897556978288216, + "tokens_seen": 1634598912 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549448345035105, + "loss": 2.5152, + "theoretical_loss": 3.4897435643544217, + "tokens_seen": 1634664448 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025493480441323975, + "loss": 2.5636, + "theoretical_loss": 3.4897314315026593, + "tokens_seen": 1634729984 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549247743229689, + "loss": 2.6746, + "theoretical_loss": 3.4897192992734776, + "tokens_seen": 1634795520 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549147442326981, + "loss": 2.6329, + "theoretical_loss": 3.4897071676668197, + "tokens_seen": 1634861056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549047141424273, + "loss": 2.5526, + "theoretical_loss": 3.4896950366826287, + "tokens_seen": 1634926592 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548946840521565, + "loss": 2.4244, + "theoretical_loss": 3.4896829063208474, + "tokens_seen": 1634992128 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025488465396188566, + "loss": 2.5959, + "theoretical_loss": 3.4896707765814194, + "tokens_seen": 1635057664 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 902458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.477343797683716, + "objective/train/theoretical_loss": 3.489658647464288, + "objective/train/tokens_used": 1655583200, + "theoretical_loss": 3.489658647464288, + "tokens_seen": 1635123200 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025487462387161484, + "loss": 2.523, + "theoretical_loss": 3.489658647464288, + "tokens_seen": 1635123200 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254864593781344, + "loss": 2.637, + "theoretical_loss": 3.489646518969395, + "tokens_seen": 1635188736 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025485456369107325, + "loss": 2.6359, + "theoretical_loss": 3.4896343910966854, + "tokens_seen": 1635254272 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548445336008024, + "loss": 2.3162, + "theoretical_loss": 3.4896222638461007, + "tokens_seen": 1635319808 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548345035105316, + "loss": 2.6989, + "theoretical_loss": 3.4896101372175856, + "tokens_seen": 1635385344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548244734202608, + "loss": 2.5139, + "theoretical_loss": 3.489598011211082, + "tokens_seen": 1635450880 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025481444332999, + "loss": 2.6453, + "theoretical_loss": 3.489585885826533, + "tokens_seen": 1635516416 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025480441323971916, + "loss": 2.7129, + "theoretical_loss": 3.489573761063883, + "tokens_seen": 1635581952 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025479438314944834, + "loss": 2.6297, + "theoretical_loss": 3.4895616369230744, + "tokens_seen": 1635647488 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547843530591775, + "loss": 2.6764, + "theoretical_loss": 3.48954951340405, + "tokens_seen": 1635713024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025477432296890676, + "loss": 2.4747, + "theoretical_loss": 3.4895373905067544, + "tokens_seen": 1635778560 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547642928786359, + "loss": 2.6663, + "theoretical_loss": 3.4895252682311293, + "tokens_seen": 1635844096 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547542627883651, + "loss": 2.5264, + "theoretical_loss": 3.489513146577119, + "tokens_seen": 1635909632 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025474423269809425, + "loss": 2.5346, + "theoretical_loss": 3.4895010255446657, + "tokens_seen": 1635975168 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547342026078235, + "loss": 2.6398, + "theoretical_loss": 3.4894889051337135, + "tokens_seen": 1636040704 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025472417251755266, + "loss": 2.5351, + "theoretical_loss": 3.489476785344205, + "tokens_seen": 1636106240 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025471414242728184, + "loss": 2.7657, + "theoretical_loss": 3.489464666176084, + "tokens_seen": 1636171776 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254704112337011, + "loss": 2.864, + "theoretical_loss": 3.4894525476292935, + "tokens_seen": 1636237312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025469408224674026, + "loss": 2.6022, + "theoretical_loss": 3.4894404297037767, + "tokens_seen": 1636302848 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546840521564694, + "loss": 2.7555, + "theoretical_loss": 3.4894283123994767, + "tokens_seen": 1636368384 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546740220661986, + "loss": 2.6694, + "theoretical_loss": 3.489416195716337, + "tokens_seen": 1636433920 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025466399197592775, + "loss": 2.425, + "theoretical_loss": 3.4894040796543013, + "tokens_seen": 1636499456 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254653961885657, + "loss": 2.4242, + "theoretical_loss": 3.489391964213312, + "tokens_seen": 1636564992 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025464393179538616, + "loss": 2.6197, + "theoretical_loss": 3.489379849393314, + "tokens_seen": 1636630528 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025463390170511535, + "loss": 2.6403, + "theoretical_loss": 3.4893677351942483, + "tokens_seen": 1636696064 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 903426, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7221128940582275, + "objective/train/theoretical_loss": 3.48935562161606, + "objective/train/tokens_used": 1657221600, + "theoretical_loss": 3.48935562161606, + "tokens_seen": 1636761600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025462387161484453, + "loss": 2.6297, + "theoretical_loss": 3.48935562161606, + "tokens_seen": 1636761600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546138415245737, + "loss": 2.5678, + "theoretical_loss": 3.489343508658692, + "tokens_seen": 1636827136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546038114343029, + "loss": 2.5796, + "theoretical_loss": 3.489331396322087, + "tokens_seen": 1636892672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545937813440321, + "loss": 2.7055, + "theoretical_loss": 3.489319284606189, + "tokens_seen": 1636958208 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025458375125376125, + "loss": 2.4974, + "theoretical_loss": 3.489307173510941, + "tokens_seen": 1637023744 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545737211634905, + "loss": 2.7415, + "theoretical_loss": 3.4892950630362867, + "tokens_seen": 1637089280 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545636910732196, + "loss": 2.647, + "theoretical_loss": 3.489282953182169, + "tokens_seen": 1637154816 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025455366098294885, + "loss": 2.6432, + "theoretical_loss": 3.489270843948532, + "tokens_seen": 1637220352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025454363089267803, + "loss": 2.6731, + "theoretical_loss": 3.489258735335318, + "tokens_seen": 1637285888 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545336008024072, + "loss": 2.7159, + "theoretical_loss": 3.489246627342471, + "tokens_seen": 1637351424 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545235707121364, + "loss": 2.5777, + "theoretical_loss": 3.4892345199699344, + "tokens_seen": 1637416960 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025451354062186563, + "loss": 2.6603, + "theoretical_loss": 3.489222413217652, + "tokens_seen": 1637482496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545035105315948, + "loss": 2.6612, + "theoretical_loss": 3.4892103070855662, + "tokens_seen": 1637548032 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254493480441324, + "loss": 2.7114, + "theoretical_loss": 3.489198201573621, + "tokens_seen": 1637613568 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025448345035105317, + "loss": 2.6173, + "theoretical_loss": 3.48918609668176, + "tokens_seen": 1637679104 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025447342026078235, + "loss": 2.5193, + "theoretical_loss": 3.4891739924099268, + "tokens_seen": 1637744640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544633901705116, + "loss": 2.4744, + "theoretical_loss": 3.4891618887580638, + "tokens_seen": 1637810176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544533600802407, + "loss": 2.6522, + "theoretical_loss": 3.489149785726115, + "tokens_seen": 1637875712 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025444332998996995, + "loss": 2.7144, + "theoretical_loss": 3.4891376833140244, + "tokens_seen": 1637941248 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544332998996991, + "loss": 2.6649, + "theoretical_loss": 3.489125581521735, + "tokens_seen": 1638006784 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544232698094283, + "loss": 2.5846, + "theoretical_loss": 3.4891134803491894, + "tokens_seen": 1638072320 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544132397191575, + "loss": 2.6829, + "theoretical_loss": 3.489101379796333, + "tokens_seen": 1638137856 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544032096288867, + "loss": 2.6106, + "theoretical_loss": 3.4890892798631077, + "tokens_seen": 1638203392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025439317953861586, + "loss": 2.5666, + "theoretical_loss": 3.489077180549457, + "tokens_seen": 1638268928 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025438314944834504, + "loss": 2.8759, + "theoretical_loss": 3.4890650818553257, + "tokens_seen": 1638334464 + }, + { + "debugging/Self-BLEU-5": 0.4993282995332528, + "debugging/distinct-1-grams": 0.7782354038853119, + "debugging/distinct-2-grams": 0.945600607571183, + "debugging/entropy-1-grams": 5.791543305262103, + "debugging/entropy-2-grams": 6.444393817535788, + "debugging/length": 527.3, + "debugging/num_segments": 10, + "epoch": 0.5, + "objective/train/docs_used": 904059, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.070502758026123, + "objective/train/theoretical_loss": 3.489052983780656, + "objective/train/tokens_used": 1658860000, + "theoretical_loss": 3.489052983780656, + "tokens_seen": 1638400000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543731193580742, + "loss": 2.8441, + "theoretical_loss": 3.489052983780656, + "tokens_seen": 1638400000 + } + ], + "max_steps": 50354, + "num_train_epochs": 9223372036854775807, + "total_flos": 8.361345024e+17, + "trial_name": null, + "trial_params": null +}