| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.01985939548000159, | |
| "global_step": 1000, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.92063492063492e-07, | |
| "loss": 10.8568, | |
| "theoretical_loss": 20.812814784551147, | |
| "tokens_seen": 65536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.984126984126984e-06, | |
| "loss": 10.8727, | |
| "theoretical_loss": 17.56619808733264, | |
| "tokens_seen": 131072 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.9761904761904763e-06, | |
| "loss": 10.7356, | |
| "theoretical_loss": 15.939474075840563, | |
| "tokens_seen": 196608 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.968253968253968e-06, | |
| "loss": 10.7113, | |
| "theoretical_loss": 14.892313738992565, | |
| "tokens_seen": 262144 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.96031746031746e-06, | |
| "loss": 10.3498, | |
| "theoretical_loss": 14.13621392076697, | |
| "tokens_seen": 327680 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.9523809523809525e-06, | |
| "loss": 10.1826, | |
| "theoretical_loss": 13.552558455554218, | |
| "tokens_seen": 393216 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.944444444444444e-06, | |
| "loss": 10.0176, | |
| "theoretical_loss": 13.081805984405184, | |
| "tokens_seen": 458752 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.936507936507936e-06, | |
| "loss": 9.6515, | |
| "theoretical_loss": 12.690126608487319, | |
| "tokens_seen": 524288 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.928571428571428e-06, | |
| "loss": 9.2598, | |
| "theoretical_loss": 12.356589446877619, | |
| "tokens_seen": 589824 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.92063492063492e-06, | |
| "loss": 9.1219, | |
| "theoretical_loss": 12.067409590039073, | |
| "tokens_seen": 655360 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.0912698412698412e-05, | |
| "loss": 9.2934, | |
| "theoretical_loss": 11.813063214105672, | |
| "tokens_seen": 720896 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.1904761904761905e-05, | |
| "loss": 9.0973, | |
| "theoretical_loss": 11.586716191710725, | |
| "tokens_seen": 786432 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.2896825396825396e-05, | |
| "loss": 9.4804, | |
| "theoretical_loss": 11.383311123190783, | |
| "tokens_seen": 851968 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.3888888888888888e-05, | |
| "loss": 9.3208, | |
| "theoretical_loss": 11.199008685115867, | |
| "tokens_seen": 917504 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.4880952380952381e-05, | |
| "loss": 9.1864, | |
| "theoretical_loss": 11.030830900981908, | |
| "tokens_seen": 983040 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.5873015873015872e-05, | |
| "loss": 9.398, | |
| "theoretical_loss": 10.876425069460945, | |
| "tokens_seen": 1048576 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.6865079365079364e-05, | |
| "loss": 8.9394, | |
| "theoretical_loss": 10.73390272306672, | |
| "tokens_seen": 1114112 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.7857142857142855e-05, | |
| "loss": 9.0636, | |
| "theoretical_loss": 10.601726859234274, | |
| "tokens_seen": 1179648 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.884920634920635e-05, | |
| "loss": 9.1583, | |
| "theoretical_loss": 10.478631155360636, | |
| "tokens_seen": 1245184 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.984126984126984e-05, | |
| "loss": 9.1124, | |
| "theoretical_loss": 10.363560926767326, | |
| "tokens_seen": 1310720 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.0833333333333333e-05, | |
| "loss": 9.0087, | |
| "theoretical_loss": 10.25562920390074, | |
| "tokens_seen": 1376256 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.1825396825396824e-05, | |
| "loss": 9.0788, | |
| "theoretical_loss": 10.154083536274014, | |
| "tokens_seen": 1441792 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.2817460317460315e-05, | |
| "loss": 8.4406, | |
| "theoretical_loss": 10.058280544736593, | |
| "tokens_seen": 1507328 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.380952380952381e-05, | |
| "loss": 8.9754, | |
| "theoretical_loss": 9.967666161844274, | |
| "tokens_seen": 1572864 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 12267, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 8.854658126831055, | |
| "objective/train/theoretical_loss": 9.881760109397105, | |
| "objective/train/tokens_used": 22098400, | |
| "theoretical_loss": 9.881760109397105, | |
| "tokens_seen": 1638400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.48015873015873e-05, | |
| "loss": 8.9164, | |
| "theoretical_loss": 9.881760109397105, | |
| "tokens_seen": 1638400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.5793650793650793e-05, | |
| "loss": 7.9452, | |
| "theoretical_loss": 9.800143574544556, | |
| "tokens_seen": 1703936 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.6785714285714284e-05, | |
| "loss": 8.81, | |
| "theoretical_loss": 9.722449329911441, | |
| "tokens_seen": 1769472 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.7777777777777776e-05, | |
| "loss": 9.0741, | |
| "theoretical_loss": 9.648353742085542, | |
| "tokens_seen": 1835008 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.876984126984127e-05, | |
| "loss": 8.8256, | |
| "theoretical_loss": 9.577570254149634, | |
| "tokens_seen": 1900544 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.9761904761904762e-05, | |
| "loss": 8.3274, | |
| "theoretical_loss": 9.509844029768848, | |
| "tokens_seen": 1966080 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.075396825396825e-05, | |
| "loss": 8.55, | |
| "theoretical_loss": 9.44494752063593, | |
| "tokens_seen": 2031616 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.1746031746031745e-05, | |
| "loss": 8.0979, | |
| "theoretical_loss": 9.382676773914453, | |
| "tokens_seen": 2097152 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.273809523809524e-05, | |
| "loss": 8.3374, | |
| "theoretical_loss": 9.322848337237977, | |
| "tokens_seen": 2162688 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.373015873015873e-05, | |
| "loss": 8.5177, | |
| "theoretical_loss": 9.26529664966427, | |
| "tokens_seen": 2228224 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.472222222222222e-05, | |
| "loss": 8.6686, | |
| "theoretical_loss": 9.209871830448751, | |
| "tokens_seen": 2293760 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.571428571428571e-05, | |
| "loss": 8.675, | |
| "theoretical_loss": 9.156437795512288, | |
| "tokens_seen": 2359296 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.670634920634921e-05, | |
| "loss": 8.4694, | |
| "theoretical_loss": 9.104870645417346, | |
| "tokens_seen": 2424832 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.76984126984127e-05, | |
| "loss": 8.2407, | |
| "theoretical_loss": 9.05505727953773, | |
| "tokens_seen": 2490368 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.8690476190476195e-05, | |
| "loss": 8.5934, | |
| "theoretical_loss": 9.006894199647824, | |
| "tokens_seen": 2555904 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.968253968253968e-05, | |
| "loss": 7.8491, | |
| "theoretical_loss": 8.960286472913351, | |
| "tokens_seen": 2621440 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.067460317460318e-05, | |
| "loss": 7.8615, | |
| "theoretical_loss": 8.915146829644607, | |
| "tokens_seen": 2686976 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.1666666666666665e-05, | |
| "loss": 7.7589, | |
| "theoretical_loss": 8.871394875482219, | |
| "tokens_seen": 2752512 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.265873015873016e-05, | |
| "loss": 8.0014, | |
| "theoretical_loss": 8.828956401157495, | |
| "tokens_seen": 2818048 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.365079365079365e-05, | |
| "loss": 7.8309, | |
| "theoretical_loss": 8.787762775782408, | |
| "tokens_seen": 2883584 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.464285714285714e-05, | |
| "loss": 8.259, | |
| "theoretical_loss": 8.747750411915451, | |
| "tokens_seen": 2949120 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.563492063492063e-05, | |
| "loss": 8.1107, | |
| "theoretical_loss": 8.708860292524829, | |
| "tokens_seen": 3014656 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.6626984126984126e-05, | |
| "loss": 8.2947, | |
| "theoretical_loss": 8.671037551512843, | |
| "tokens_seen": 3080192 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.761904761904762e-05, | |
| "loss": 7.764, | |
| "theoretical_loss": 8.63423110073947, | |
| "tokens_seen": 3145728 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.8611111111111115e-05, | |
| "loss": 8.1678, | |
| "theoretical_loss": 8.59839329754032, | |
| "tokens_seen": 3211264 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 13483, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 8.108365058898926, | |
| "objective/train/theoretical_loss": 8.563479647615063, | |
| "objective/train/tokens_used": 23736800, | |
| "theoretical_loss": 8.563479647615063, | |
| "tokens_seen": 3276800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.96031746031746e-05, | |
| "loss": 7.5801, | |
| "theoretical_loss": 8.563479647615063, | |
| "tokens_seen": 3276800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.05952380952381e-05, | |
| "loss": 7.8434, | |
| "theoretical_loss": 8.52944853889911, | |
| "tokens_seen": 3342336 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.1587301587301586e-05, | |
| "loss": 7.8492, | |
| "theoretical_loss": 8.496261002649998, | |
| "tokens_seen": 3407872 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.257936507936508e-05, | |
| "loss": 7.8312, | |
| "theoretical_loss": 8.463880498501183, | |
| "tokens_seen": 3473408 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.357142857142857e-05, | |
| "loss": 7.3015, | |
| "theoretical_loss": 8.432272720676774, | |
| "tokens_seen": 3538944 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.4563492063492063e-05, | |
| "loss": 7.9474, | |
| "theoretical_loss": 8.401405422934712, | |
| "tokens_seen": 3604480 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.555555555555555e-05, | |
| "loss": 7.6148, | |
| "theoretical_loss": 8.371248260124204, | |
| "tokens_seen": 3670016 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.6547619047619046e-05, | |
| "loss": 7.5108, | |
| "theoretical_loss": 8.341772644515071, | |
| "tokens_seen": 3735552 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.753968253968254e-05, | |
| "loss": 7.6168, | |
| "theoretical_loss": 8.312951615289325, | |
| "tokens_seen": 3801088 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.8531746031746036e-05, | |
| "loss": 7.8436, | |
| "theoretical_loss": 8.284759719785177, | |
| "tokens_seen": 3866624 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.9523809523809524e-05, | |
| "loss": 7.0651, | |
| "theoretical_loss": 8.257172905255858, | |
| "tokens_seen": 3932160 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.051587301587302e-05, | |
| "loss": 7.4859, | |
| "theoretical_loss": 8.23016842005411, | |
| "tokens_seen": 3997696 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.15079365079365e-05, | |
| "loss": 7.531, | |
| "theoretical_loss": 8.203724723281965, | |
| "tokens_seen": 4063232 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.25e-05, | |
| "loss": 7.5815, | |
| "theoretical_loss": 8.177821402057042, | |
| "tokens_seen": 4128768 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.349206349206349e-05, | |
| "loss": 7.2471, | |
| "theoretical_loss": 8.152439095643611, | |
| "tokens_seen": 4194304 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.448412698412699e-05, | |
| "loss": 7.4488, | |
| "theoretical_loss": 8.127559425781495, | |
| "tokens_seen": 4259840 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.547619047619048e-05, | |
| "loss": 7.4517, | |
| "theoretical_loss": 8.103164932619705, | |
| "tokens_seen": 4325376 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.646825396825397e-05, | |
| "loss": 7.1039, | |
| "theoretical_loss": 8.079239015726635, | |
| "tokens_seen": 4390912 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.746031746031745e-05, | |
| "loss": 7.1279, | |
| "theoretical_loss": 8.055765879705412, | |
| "tokens_seen": 4456448 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.845238095238096e-05, | |
| "loss": 7.3354, | |
| "theoretical_loss": 8.032730483993003, | |
| "tokens_seen": 4521984 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.944444444444444e-05, | |
| "loss": 7.1485, | |
| "theoretical_loss": 8.01011849646583, | |
| "tokens_seen": 4587520 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.043650793650793e-05, | |
| "loss": 6.9851, | |
| "theoretical_loss": 7.987916250513374, | |
| "tokens_seen": 4653056 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.142857142857142e-05, | |
| "loss": 6.9432, | |
| "theoretical_loss": 7.966110705275796, | |
| "tokens_seen": 4718592 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.242063492063492e-05, | |
| "loss": 6.9933, | |
| "theoretical_loss": 7.944689408771984, | |
| "tokens_seen": 4784128 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.341269841269842e-05, | |
| "loss": 7.0325, | |
| "theoretical_loss": 7.923640463671514, | |
| "tokens_seen": 4849664 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 14033, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 6.057826519012451, | |
| "objective/train/theoretical_loss": 7.902952495488062, | |
| "objective/train/tokens_used": 25375200, | |
| "theoretical_loss": 7.902952495488062, | |
| "tokens_seen": 4915200 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.440476190476191e-05, | |
| "loss": 6.7791, | |
| "theoretical_loss": 7.902952495488062, | |
| "tokens_seen": 4915200 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.53968253968254e-05, | |
| "loss": 6.7978, | |
| "theoretical_loss": 7.882614622993199, | |
| "tokens_seen": 4980736 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.63888888888889e-05, | |
| "loss": 6.8558, | |
| "theoretical_loss": 7.8626164306686235, | |
| "tokens_seen": 5046272 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.738095238095239e-05, | |
| "loss": 6.9141, | |
| "theoretical_loss": 7.842947943031932, | |
| "tokens_seen": 5111808 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.837301587301588e-05, | |
| "loss": 6.9255, | |
| "theoretical_loss": 7.8235996006863076, | |
| "tokens_seen": 5177344 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.936507936507937e-05, | |
| "loss": 6.7895, | |
| "theoretical_loss": 7.8045622379581605, | |
| "tokens_seen": 5242880 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.035714285714287e-05, | |
| "loss": 6.7059, | |
| "theoretical_loss": 7.785827061999068, | |
| "tokens_seen": 5308416 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.134920634920635e-05, | |
| "loss": 6.815, | |
| "theoretical_loss": 7.767385633239359, | |
| "tokens_seen": 5373952 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.234126984126984e-05, | |
| "loss": 6.8445, | |
| "theoretical_loss": 7.749229847090614, | |
| "tokens_seen": 5439488 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.333333333333333e-05, | |
| "loss": 6.7849, | |
| "theoretical_loss": 7.731351916803314, | |
| "tokens_seen": 5505024 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.432539682539683e-05, | |
| "loss": 6.5349, | |
| "theoretical_loss": 7.7137443573939155, | |
| "tokens_seen": 5570560 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.531746031746032e-05, | |
| "loss": 6.5103, | |
| "theoretical_loss": 7.696399970562929, | |
| "tokens_seen": 5636096 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.630952380952381e-05, | |
| "loss": 6.9307, | |
| "theoretical_loss": 7.679311830532177, | |
| "tokens_seen": 5701632 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.73015873015873e-05, | |
| "loss": 6.8115, | |
| "theoretical_loss": 7.662473270735324, | |
| "tokens_seen": 5767168 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.82936507936508e-05, | |
| "loss": 6.6601, | |
| "theoretical_loss": 7.645877871301275, | |
| "tokens_seen": 5832704 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.928571428571429e-05, | |
| "loss": 6.2238, | |
| "theoretical_loss": 7.6295194472748555, | |
| "tokens_seen": 5898240 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.027777777777777e-05, | |
| "loss": 6.6176, | |
| "theoretical_loss": 7.613392037523692, | |
| "tokens_seen": 5963776 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.126984126984126e-05, | |
| "loss": 6.5932, | |
| "theoretical_loss": 7.597489894284275, | |
| "tokens_seen": 6029312 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.226190476190476e-05, | |
| "loss": 6.51, | |
| "theoretical_loss": 7.581807473303883, | |
| "tokens_seen": 6094848 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.325396825396825e-05, | |
| "loss": 6.3768, | |
| "theoretical_loss": 7.566339424538366, | |
| "tokens_seen": 6160384 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.424603174603175e-05, | |
| "loss": 6.5465, | |
| "theoretical_loss": 7.551080583368945, | |
| "tokens_seen": 6225920 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.523809523809524e-05, | |
| "loss": 6.8931, | |
| "theoretical_loss": 7.536025962303915, | |
| "tokens_seen": 6291456 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.623015873015874e-05, | |
| "loss": 6.5266, | |
| "theoretical_loss": 7.521170743133757, | |
| "tokens_seen": 6356992 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.722222222222223e-05, | |
| "loss": 6.4856, | |
| "theoretical_loss": 7.506510269510493, | |
| "tokens_seen": 6422528 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.821428571428572e-05, | |
| "loss": 6.4763, | |
| "theoretical_loss": 7.492040039924245, | |
| "tokens_seen": 6488064 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 15357, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 6.699707984924316, | |
| "objective/train/theoretical_loss": 7.4777557010520255, | |
| "objective/train/tokens_used": 27013600, | |
| "theoretical_loss": 7.4777557010520255, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.92063492063492e-05, | |
| "loss": 6.6532, | |
| "theoretical_loss": 7.4777557010520255, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010019841269841271, | |
| "loss": 6.4, | |
| "theoretical_loss": 7.463653041455457, | |
| "tokens_seen": 6619136 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001011904761904762, | |
| "loss": 6.2462, | |
| "theoretical_loss": 7.449727985605911, | |
| "tokens_seen": 6684672 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010218253968253968, | |
| "loss": 6.6399, | |
| "theoretical_loss": 7.435976588217015, | |
| "tokens_seen": 6750208 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010317460317460317, | |
| "loss": 6.1887, | |
| "theoretical_loss": 7.4223950288659, | |
| "tokens_seen": 6815744 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010416666666666667, | |
| "loss": 6.4129, | |
| "theoretical_loss": 7.40897960688587, | |
| "tokens_seen": 6881280 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010515873015873016, | |
| "loss": 6.2296, | |
| "theoretical_loss": 7.395726736514341, | |
| "tokens_seen": 6946816 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010615079365079365, | |
| "loss": 6.3505, | |
| "theoretical_loss": 7.3826329422810275, | |
| "tokens_seen": 7012352 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010714285714285714, | |
| "loss": 6.5043, | |
| "theoretical_loss": 7.369694854622368, | |
| "tokens_seen": 7077888 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010813492063492064, | |
| "loss": 6.3689, | |
| "theoretical_loss": 7.3569092057090835, | |
| "tokens_seen": 7143424 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010912698412698413, | |
| "loss": 6.5583, | |
| "theoretical_loss": 7.3442728254746825, | |
| "tokens_seen": 7208960 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011011904761904761, | |
| "loss": 6.3757, | |
| "theoretical_loss": 7.3317826378335145, | |
| "tokens_seen": 7274496 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001111111111111111, | |
| "loss": 6.3164, | |
| "theoretical_loss": 7.319435657077673, | |
| "tokens_seen": 7340032 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001121031746031746, | |
| "loss": 6.205, | |
| "theoretical_loss": 7.307228984442819, | |
| "tokens_seen": 7405568 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011309523809523809, | |
| "loss": 6.4975, | |
| "theoretical_loss": 7.29515980483356, | |
| "tokens_seen": 7471104 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011408730158730158, | |
| "loss": 6.087, | |
| "theoretical_loss": 7.283225383699648, | |
| "tokens_seen": 7536640 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011507936507936508, | |
| "loss": 6.0785, | |
| "theoretical_loss": 7.271423064054828, | |
| "tokens_seen": 7602176 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011607142857142858, | |
| "loss": 6.4949, | |
| "theoretical_loss": 7.259750263630618, | |
| "tokens_seen": 7667712 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011706349206349207, | |
| "loss": 6.2155, | |
| "theoretical_loss": 7.248204472157866, | |
| "tokens_seen": 7733248 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011805555555555556, | |
| "loss": 6.0229, | |
| "theoretical_loss": 7.236783248769257, | |
| "tokens_seen": 7798784 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011904761904761905, | |
| "loss": 6.6244, | |
| "theoretical_loss": 7.225484219516492, | |
| "tokens_seen": 7864320 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012003968253968255, | |
| "loss": 5.9387, | |
| "theoretical_loss": 7.2143050749961155, | |
| "tokens_seen": 7929856 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012103174603174604, | |
| "loss": 6.2842, | |
| "theoretical_loss": 7.2032435680783955, | |
| "tokens_seen": 7995392 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012202380952380953, | |
| "loss": 6.2047, | |
| "theoretical_loss": 7.19229751173401, | |
| "tokens_seen": 8060928 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.000123015873015873, | |
| "loss": 6.2571, | |
| "theoretical_loss": 7.181464776953511, | |
| "tokens_seen": 8126464 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 16711, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 6.2001729011535645, | |
| "objective/train/theoretical_loss": 7.1707432907549595, | |
| "objective/train/tokens_used": 28652000, | |
| "theoretical_loss": 7.1707432907549595, | |
| "tokens_seen": 8192000 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001240079365079365, | |
| "loss": 6.0533, | |
| "theoretical_loss": 7.1707432907549595, | |
| "tokens_seen": 8192000 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.000125, | |
| "loss": 6.0526, | |
| "theoretical_loss": 7.160131034275267, | |
| "tokens_seen": 8257536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001259920634920635, | |
| "loss": 6.0532, | |
| "theoretical_loss": 7.149626040941133, | |
| "tokens_seen": 8323072 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012698412698412698, | |
| "loss": 5.9719, | |
| "theoretical_loss": 7.139226394715633, | |
| "tokens_seen": 8388608 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012797619047619048, | |
| "loss": 6.0403, | |
| "theoretical_loss": 7.12893022841679, | |
| "tokens_seen": 8454144 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012896825396825398, | |
| "loss": 5.7847, | |
| "theoretical_loss": 7.118735722104612, | |
| "tokens_seen": 8519680 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012996031746031748, | |
| "loss": 5.9094, | |
| "theoretical_loss": 7.10864110153334, | |
| "tokens_seen": 8585216 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00013095238095238096, | |
| "loss": 6.0872, | |
| "theoretical_loss": 7.098644636665766, | |
| "tokens_seen": 8650752 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00013194444444444446, | |
| "loss": 6.0185, | |
| "theoretical_loss": 7.088744640246688, | |
| "tokens_seen": 8716288 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00013293650793650793, | |
| "loss": 6.0768, | |
| "theoretical_loss": 7.078939466432745, | |
| "tokens_seen": 8781824 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00013392857142857144, | |
| "loss": 6.3022, | |
| "theoretical_loss": 7.069227509475962, | |
| "tokens_seen": 8847360 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001349206349206349, | |
| "loss": 6.0881, | |
| "theoretical_loss": 7.059607202458563, | |
| "tokens_seen": 8912896 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001359126984126984, | |
| "loss": 5.7381, | |
| "theoretical_loss": 7.0500770160766635, | |
| "tokens_seen": 8978432 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001369047619047619, | |
| "loss": 5.9916, | |
| "theoretical_loss": 7.04063545747062, | |
| "tokens_seen": 9043968 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00013789682539682541, | |
| "loss": 6.0983, | |
| "theoretical_loss": 7.031281069099929, | |
| "tokens_seen": 9109504 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001388888888888889, | |
| "loss": 6.0662, | |
| "theoretical_loss": 7.022012427660673, | |
| "tokens_seen": 9175040 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001398809523809524, | |
| "loss": 5.7078, | |
| "theoretical_loss": 7.012828143043604, | |
| "tokens_seen": 9240576 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014087301587301586, | |
| "loss": 6.2455, | |
| "theoretical_loss": 7.003726857331067, | |
| "tokens_seen": 9306112 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014186507936507937, | |
| "loss": 5.9609, | |
| "theoretical_loss": 6.994707243831053, | |
| "tokens_seen": 9371648 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014285714285714284, | |
| "loss": 5.9423, | |
| "theoretical_loss": 6.985768006146776, | |
| "tokens_seen": 9437184 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014384920634920634, | |
| "loss": 6.0329, | |
| "theoretical_loss": 6.976907877280185, | |
| "tokens_seen": 9502720 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014484126984126984, | |
| "loss": 5.7799, | |
| "theoretical_loss": 6.96812561876801, | |
| "tokens_seen": 9568256 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014583333333333335, | |
| "loss": 5.9063, | |
| "theoretical_loss": 6.95942001984889, | |
| "tokens_seen": 9633792 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014682539682539685, | |
| "loss": 5.9317, | |
| "theoretical_loss": 6.950789896660304, | |
| "tokens_seen": 9699328 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014781746031746032, | |
| "loss": 5.8925, | |
| "theoretical_loss": 6.942234091464025, | |
| "tokens_seen": 9764864 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 17166, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 6.479184627532959, | |
| "objective/train/theoretical_loss": 6.933751471898896, | |
| "objective/train/tokens_used": 30290400, | |
| "theoretical_loss": 6.933751471898896, | |
| "tokens_seen": 9830400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014880952380952382, | |
| "loss": 6.2539, | |
| "theoretical_loss": 6.933751471898896, | |
| "tokens_seen": 9830400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001498015873015873, | |
| "loss": 5.996, | |
| "theoretical_loss": 6.925340930259813, | |
| "tokens_seen": 9895936 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001507936507936508, | |
| "loss": 5.8197, | |
| "theoretical_loss": 6.917001382801793, | |
| "tokens_seen": 9961472 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015178571428571427, | |
| "loss": 6.0534, | |
| "theoretical_loss": 6.908731769068142, | |
| "tokens_seen": 10027008 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001527777777777778, | |
| "loss": 5.7838, | |
| "theoretical_loss": 6.900531051241684, | |
| "tokens_seen": 10092544 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015376984126984128, | |
| "loss": 5.7799, | |
| "theoretical_loss": 6.892398213518156, | |
| "tokens_seen": 10158080 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015476190476190478, | |
| "loss": 6.1056, | |
| "theoretical_loss": 6.884332261500866, | |
| "tokens_seen": 10223616 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015575396825396825, | |
| "loss": 5.5432, | |
| "theoretical_loss": 6.876332221615746, | |
| "tokens_seen": 10289152 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015674603174603175, | |
| "loss": 5.8572, | |
| "theoretical_loss": 6.8683971405459925, | |
| "tokens_seen": 10354688 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015773809523809523, | |
| "loss": 6.226, | |
| "theoretical_loss": 6.860526084685546, | |
| "tokens_seen": 10420224 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015873015873015873, | |
| "loss": 5.7117, | |
| "theoretical_loss": 6.852718139610625, | |
| "tokens_seen": 10485760 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001597222222222222, | |
| "loss": 5.6684, | |
| "theoretical_loss": 6.844972409568637, | |
| "tokens_seen": 10551296 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016071428571428573, | |
| "loss": 5.8166, | |
| "theoretical_loss": 6.8372880169837895, | |
| "tokens_seen": 10616832 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001617063492063492, | |
| "loss": 5.971, | |
| "theoretical_loss": 6.829664101978745, | |
| "tokens_seen": 10682368 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001626984126984127, | |
| "loss": 5.7343, | |
| "theoretical_loss": 6.8220998219117135, | |
| "tokens_seen": 10747904 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016369047619047618, | |
| "loss": 5.8061, | |
| "theoretical_loss": 6.81459435092839, | |
| "tokens_seen": 10813440 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016468253968253969, | |
| "loss": 6.0029, | |
| "theoretical_loss": 6.807146879528176, | |
| "tokens_seen": 10878976 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016567460317460316, | |
| "loss": 5.5367, | |
| "theoretical_loss": 6.79975661414414, | |
| "tokens_seen": 10944512 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016666666666666666, | |
| "loss": 5.6256, | |
| "theoretical_loss": 6.79242277673622, | |
| "tokens_seen": 11010048 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016765873015873016, | |
| "loss": 5.7868, | |
| "theoretical_loss": 6.785144604397143, | |
| "tokens_seen": 11075584 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016865079365079366, | |
| "loss": 5.6046, | |
| "theoretical_loss": 6.777921348970633, | |
| "tokens_seen": 11141120 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016964285714285717, | |
| "loss": 5.9753, | |
| "theoretical_loss": 6.770752276681419, | |
| "tokens_seen": 11206656 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017063492063492064, | |
| "loss": 5.9022, | |
| "theoretical_loss": 6.76363666777662, | |
| "tokens_seen": 11272192 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017162698412698414, | |
| "loss": 5.8921, | |
| "theoretical_loss": 6.756573816178118, | |
| "tokens_seen": 11337728 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017261904761904762, | |
| "loss": 5.9722, | |
| "theoretical_loss": 6.749563029145481, | |
| "tokens_seen": 11403264 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 18369, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.672413349151611, | |
| "objective/train/theoretical_loss": 6.742603626949085, | |
| "objective/train/tokens_used": 31928800, | |
| "theoretical_loss": 6.742603626949085, | |
| "tokens_seen": 11468800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017361111111111112, | |
| "loss": 5.4506, | |
| "theoretical_loss": 6.742603626949085, | |
| "tokens_seen": 11468800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001746031746031746, | |
| "loss": 6.0078, | |
| "theoretical_loss": 6.735694942553071, | |
| "tokens_seen": 11534336 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001755952380952381, | |
| "loss": 5.8091, | |
| "theoretical_loss": 6.728836321307757, | |
| "tokens_seen": 11599872 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001765873015873016, | |
| "loss": 5.6113, | |
| "theoretical_loss": 6.722027120651221, | |
| "tokens_seen": 11665408 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001775793650793651, | |
| "loss": 5.6758, | |
| "theoretical_loss": 6.715266709819684, | |
| "tokens_seen": 11730944 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017857142857142857, | |
| "loss": 5.6378, | |
| "theoretical_loss": 6.708554469566408, | |
| "tokens_seen": 11796480 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017956349206349207, | |
| "loss": 5.7411, | |
| "theoretical_loss": 6.701889791888819, | |
| "tokens_seen": 11862016 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018055555555555555, | |
| "loss": 5.8311, | |
| "theoretical_loss": 6.695272079763555, | |
| "tokens_seen": 11927552 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018154761904761905, | |
| "loss": 5.2846, | |
| "theoretical_loss": 6.688700746889175, | |
| "tokens_seen": 11993088 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018253968253968252, | |
| "loss": 5.7687, | |
| "theoretical_loss": 6.682175217436269, | |
| "tokens_seen": 12058624 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018353174603174602, | |
| "loss": 5.7687, | |
| "theoretical_loss": 6.675694925804711, | |
| "tokens_seen": 12124160 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018452380952380953, | |
| "loss": 5.6209, | |
| "theoretical_loss": 6.66925931638781, | |
| "tokens_seen": 12189696 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018551587301587303, | |
| "loss": 5.9193, | |
| "theoretical_loss": 6.662867843343154, | |
| "tokens_seen": 12255232 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001865079365079365, | |
| "loss": 5.6754, | |
| "theoretical_loss": 6.6565199703698745, | |
| "tokens_seen": 12320768 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001875, | |
| "loss": 5.8597, | |
| "theoretical_loss": 6.6502151704921655, | |
| "tokens_seen": 12386304 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001884920634920635, | |
| "loss": 5.709, | |
| "theoretical_loss": 6.643952925848826, | |
| "tokens_seen": 12451840 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018948412698412698, | |
| "loss": 5.7054, | |
| "theoretical_loss": 6.637732727488622, | |
| "tokens_seen": 12517376 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019047619047619048, | |
| "loss": 5.5972, | |
| "theoretical_loss": 6.6315540751713, | |
| "tokens_seen": 12582912 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019146825396825398, | |
| "loss": 5.5887, | |
| "theoretical_loss": 6.625416477174044, | |
| "tokens_seen": 12648448 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019246031746031748, | |
| "loss": 5.7082, | |
| "theoretical_loss": 6.619319450103218, | |
| "tokens_seen": 12713984 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019345238095238096, | |
| "loss": 5.5405, | |
| "theoretical_loss": 6.613262518711206, | |
| "tokens_seen": 12779520 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019444444444444446, | |
| "loss": 5.5806, | |
| "theoretical_loss": 6.607245215718208, | |
| "tokens_seen": 12845056 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019543650793650793, | |
| "loss": 5.4612, | |
| "theoretical_loss": 6.601267081638806, | |
| "tokens_seen": 12910592 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019642857142857144, | |
| "loss": 5.6405, | |
| "theoretical_loss": 6.595327664613176, | |
| "tokens_seen": 12976128 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001974206349206349, | |
| "loss": 5.3968, | |
| "theoretical_loss": 6.58942652024278, | |
| "tokens_seen": 13041664 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 19017, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.398904323577881, | |
| "objective/train/theoretical_loss": 6.583563211430409, | |
| "objective/train/tokens_used": 33567200, | |
| "theoretical_loss": 6.583563211430409, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001984126984126984, | |
| "loss": 5.5392, | |
| "theoretical_loss": 6.583563211430409, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019940476190476191, | |
| "loss": 5.3995, | |
| "theoretical_loss": 6.577737308224426, | |
| "tokens_seen": 13172736 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020039682539682542, | |
| "loss": 5.5665, | |
| "theoretical_loss": 6.571948387667094, | |
| "tokens_seen": 13238272 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002013888888888889, | |
| "loss": 5.5745, | |
| "theoretical_loss": 6.566196033646859, | |
| "tokens_seen": 13303808 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002023809523809524, | |
| "loss": 5.4756, | |
| "theoretical_loss": 6.560479836754459, | |
| "tokens_seen": 13369344 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020337301587301587, | |
| "loss": 5.7895, | |
| "theoretical_loss": 6.554799394142741, | |
| "tokens_seen": 13434880 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020436507936507937, | |
| "loss": 5.3965, | |
| "theoretical_loss": 6.549154309390087, | |
| "tokens_seen": 13500416 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020535714285714284, | |
| "loss": 5.494, | |
| "theoretical_loss": 6.5435441923673245, | |
| "tokens_seen": 13565952 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020634920634920634, | |
| "loss": 5.5559, | |
| "theoretical_loss": 6.537968659108021, | |
| "tokens_seen": 13631488 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020734126984126985, | |
| "loss": 5.9125, | |
| "theoretical_loss": 6.532427331682063, | |
| "tokens_seen": 13697024 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020833333333333335, | |
| "loss": 5.5235, | |
| "theoretical_loss": 6.5269198380724145, | |
| "tokens_seen": 13762560 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020932539682539685, | |
| "loss": 5.6307, | |
| "theoretical_loss": 6.5214458120549725, | |
| "tokens_seen": 13828096 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021031746031746032, | |
| "loss": 5.3661, | |
| "theoretical_loss": 6.516004893081412, | |
| "tokens_seen": 13893632 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021130952380952382, | |
| "loss": 5.2823, | |
| "theoretical_loss": 6.510596726164945, | |
| "tokens_seen": 13959168 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002123015873015873, | |
| "loss": 5.4241, | |
| "theoretical_loss": 6.505220961768906, | |
| "tokens_seen": 14024704 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002132936507936508, | |
| "loss": 5.3679, | |
| "theoretical_loss": 6.499877255698063, | |
| "tokens_seen": 14090240 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021428571428571427, | |
| "loss": 5.2005, | |
| "theoretical_loss": 6.494565268992613, | |
| "tokens_seen": 14155776 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002152777777777778, | |
| "loss": 5.6155, | |
| "theoretical_loss": 6.489284667824741, | |
| "tokens_seen": 14221312 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021626984126984128, | |
| "loss": 5.6027, | |
| "theoretical_loss": 6.484035123397694, | |
| "tokens_seen": 14286848 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021726190476190478, | |
| "loss": 5.5959, | |
| "theoretical_loss": 6.478816311847305, | |
| "tokens_seen": 14352384 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021825396825396825, | |
| "loss": 5.7842, | |
| "theoretical_loss": 6.473627914145865, | |
| "tokens_seen": 14417920 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021924603174603176, | |
| "loss": 5.4174, | |
| "theoretical_loss": 6.468469616008303, | |
| "tokens_seen": 14483456 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022023809523809523, | |
| "loss": 5.4423, | |
| "theoretical_loss": 6.463341107800611, | |
| "tokens_seen": 14548992 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022123015873015873, | |
| "loss": 5.2413, | |
| "theoretical_loss": 6.458242084450426, | |
| "tokens_seen": 14614528 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002222222222222222, | |
| "loss": 5.4841, | |
| "theoretical_loss": 6.453172245359726, | |
| "tokens_seen": 14680064 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 20187, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.4210638999938965, | |
| "objective/train/theoretical_loss": 6.448131294319589, | |
| "objective/train/tokens_used": 35205600, | |
| "theoretical_loss": 6.448131294319589, | |
| "tokens_seen": 14745600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022321428571428573, | |
| "loss": 5.318, | |
| "theoretical_loss": 6.448131294319589, | |
| "tokens_seen": 14745600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002242063492063492, | |
| "loss": 5.5298, | |
| "theoretical_loss": 6.443118939426935, | |
| "tokens_seen": 14811136 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002251984126984127, | |
| "loss": 5.5104, | |
| "theoretical_loss": 6.438134893003209, | |
| "tokens_seen": 14876672 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022619047619047618, | |
| "loss": 5.4089, | |
| "theoretical_loss": 6.43317887151496, | |
| "tokens_seen": 14942208 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022718253968253969, | |
| "loss": 5.5783, | |
| "theoretical_loss": 6.428250595496234, | |
| "tokens_seen": 15007744 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022817460317460316, | |
| "loss": 5.4029, | |
| "theoretical_loss": 6.4233497894727805, | |
| "tokens_seen": 15073280 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022916666666666666, | |
| "loss": 5.6043, | |
| "theoretical_loss": 6.418476181887964, | |
| "tokens_seen": 15138816 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023015873015873016, | |
| "loss": 5.5123, | |
| "theoretical_loss": 6.413629505030386, | |
| "tokens_seen": 15204352 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023115079365079367, | |
| "loss": 5.5084, | |
| "theoretical_loss": 6.408809494963135, | |
| "tokens_seen": 15269888 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023214285714285717, | |
| "loss": 5.269, | |
| "theoretical_loss": 6.404015891454651, | |
| "tokens_seen": 15335424 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023313492063492064, | |
| "loss": 5.4379, | |
| "theoretical_loss": 6.399248437911128, | |
| "tokens_seen": 15400960 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023412698412698414, | |
| "loss": 5.4346, | |
| "theoretical_loss": 6.394506881310447, | |
| "tokens_seen": 15466496 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023511904761904762, | |
| "loss": 5.7191, | |
| "theoretical_loss": 6.38979097213757, | |
| "tokens_seen": 15532032 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023611111111111112, | |
| "loss": 5.5484, | |
| "theoretical_loss": 6.385100464321383, | |
| "tokens_seen": 15597568 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002371031746031746, | |
| "loss": 5.4379, | |
| "theoretical_loss": 6.380435115172919, | |
| "tokens_seen": 15663104 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002380952380952381, | |
| "loss": 5.4785, | |
| "theoretical_loss": 6.375794685324962, | |
| "tokens_seen": 15728640 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002390873015873016, | |
| "loss": 5.2088, | |
| "theoretical_loss": 6.371178938672962, | |
| "tokens_seen": 15794176 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002400793650793651, | |
| "loss": 5.5076, | |
| "theoretical_loss": 6.366587642317243, | |
| "tokens_seen": 15859712 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024107142857142857, | |
| "loss": 5.6351, | |
| "theoretical_loss": 6.362020566506475, | |
| "tokens_seen": 15925248 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024206349206349207, | |
| "loss": 5.6029, | |
| "theoretical_loss": 6.3574774845823665, | |
| "tokens_seen": 15990784 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024305555555555555, | |
| "loss": 5.1969, | |
| "theoretical_loss": 6.352958172925549, | |
| "tokens_seen": 16056320 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024404761904761905, | |
| "loss": 5.285, | |
| "theoretical_loss": 6.348462410902625, | |
| "tokens_seen": 16121856 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024503968253968255, | |
| "loss": 5.6805, | |
| "theoretical_loss": 6.343989980814362, | |
| "tokens_seen": 16187392 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.000246031746031746, | |
| "loss": 5.5855, | |
| "theoretical_loss": 6.339540667844965, | |
| "tokens_seen": 16252928 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024702380952380955, | |
| "loss": 5.3559, | |
| "theoretical_loss": 6.335114260012455, | |
| "tokens_seen": 16318464 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 20737, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.463630199432373, | |
| "objective/train/theoretical_loss": 6.330710548120079, | |
| "objective/train/tokens_used": 36844000, | |
| "theoretical_loss": 6.330710548120079, | |
| "tokens_seen": 16384000 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.000248015873015873, | |
| "loss": 5.515, | |
| "theoretical_loss": 6.330710548120079, | |
| "tokens_seen": 16384000 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002490079365079365, | |
| "loss": 5.3596, | |
| "theoretical_loss": 6.326329325708746, | |
| "tokens_seen": 16449536 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025, | |
| "loss": 5.249, | |
| "theoretical_loss": 6.321970389010465, | |
| "tokens_seen": 16515072 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002509920634920635, | |
| "loss": 5.3675, | |
| "theoretical_loss": 6.317633536902759, | |
| "tokens_seen": 16580608 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000251984126984127, | |
| "loss": 5.0919, | |
| "theoretical_loss": 6.313318570864016, | |
| "tokens_seen": 16646144 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025297619047619046, | |
| "loss": 5.1228, | |
| "theoretical_loss": 6.30902529492978, | |
| "tokens_seen": 16711680 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025396825396825396, | |
| "loss": 5.4573, | |
| "theoretical_loss": 6.304753515649935, | |
| "tokens_seen": 16777216 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025496031746031746, | |
| "loss": 5.1499, | |
| "theoretical_loss": 6.30050304204677, | |
| "tokens_seen": 16842752 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025595238095238096, | |
| "loss": 5.6432, | |
| "theoretical_loss": 6.296273685573913, | |
| "tokens_seen": 16908288 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002569444444444444, | |
| "loss": 5.439, | |
| "theoretical_loss": 6.292065260076094, | |
| "tokens_seen": 16973824 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025793650793650796, | |
| "loss": 5.2959, | |
| "theoretical_loss": 6.287877581749726, | |
| "tokens_seen": 17039360 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025892857142857146, | |
| "loss": 5.3502, | |
| "theoretical_loss": 6.2837104691042915, | |
| "tokens_seen": 17104896 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025992063492063497, | |
| "loss": 5.5134, | |
| "theoretical_loss": 6.279563742924502, | |
| "tokens_seen": 17170432 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002609126984126984, | |
| "loss": 5.4871, | |
| "theoretical_loss": 6.275437226233224, | |
| "tokens_seen": 17235968 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002619047619047619, | |
| "loss": 5.3132, | |
| "theoretical_loss": 6.271330744255137, | |
| "tokens_seen": 17301504 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002628968253968254, | |
| "loss": 5.1796, | |
| "theoretical_loss": 6.267244124381133, | |
| "tokens_seen": 17367040 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002638888888888889, | |
| "loss": 5.2656, | |
| "theoretical_loss": 6.2631771961334035, | |
| "tokens_seen": 17432576 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00026488095238095237, | |
| "loss": 5.5896, | |
| "theoretical_loss": 6.259129791131242, | |
| "tokens_seen": 17498112 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00026587301587301587, | |
| "loss": 5.14, | |
| "theoretical_loss": 6.255101743057493, | |
| "tokens_seen": 17563648 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00026686507936507937, | |
| "loss": 5.3606, | |
| "theoretical_loss": 6.251092887625685, | |
| "tokens_seen": 17629184 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00026785714285714287, | |
| "loss": 5.487, | |
| "theoretical_loss": 6.247103062547796, | |
| "tokens_seen": 17694720 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002688492063492063, | |
| "loss": 5.2887, | |
| "theoretical_loss": 6.243132107502647, | |
| "tokens_seen": 17760256 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002698412698412698, | |
| "loss": 5.4254, | |
| "theoretical_loss": 6.239179864104911, | |
| "tokens_seen": 17825792 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002708333333333333, | |
| "loss": 5.4668, | |
| "theoretical_loss": 6.235246175874727, | |
| "tokens_seen": 17891328 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002718253968253968, | |
| "loss": 5.3584, | |
| "theoretical_loss": 6.231330888207894, | |
| "tokens_seen": 17956864 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 21782, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.119491100311279, | |
| "objective/train/theoretical_loss": 6.227433848346639, | |
| "objective/train/tokens_used": 38482400, | |
| "theoretical_loss": 6.227433848346639, | |
| "tokens_seen": 18022400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002728174603174603, | |
| "loss": 5.1069, | |
| "theoretical_loss": 6.227433848346639, | |
| "tokens_seen": 18022400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002738095238095238, | |
| "loss": 5.1568, | |
| "theoretical_loss": 6.22355490535095, | |
| "tokens_seen": 18087936 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002748015873015873, | |
| "loss": 5.4409, | |
| "theoretical_loss": 6.219693910070452, | |
| "tokens_seen": 18153472 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00027579365079365083, | |
| "loss": 5.1939, | |
| "theoretical_loss": 6.215850715116817, | |
| "tokens_seen": 18219008 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00027678571428571433, | |
| "loss": 5.2274, | |
| "theoretical_loss": 6.212025174836697, | |
| "tokens_seen": 18284544 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002777777777777778, | |
| "loss": 5.5299, | |
| "theoretical_loss": 6.208217145285173, | |
| "tokens_seen": 18350080 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002787698412698413, | |
| "loss": 5.4441, | |
| "theoretical_loss": 6.204426484199696, | |
| "tokens_seen": 18415616 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002797619047619048, | |
| "loss": 5.3135, | |
| "theoretical_loss": 6.200653050974525, | |
| "tokens_seen": 18481152 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002807539682539683, | |
| "loss": 5.189, | |
| "theoretical_loss": 6.196896706635635, | |
| "tokens_seen": 18546688 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00028174603174603173, | |
| "loss": 5.0786, | |
| "theoretical_loss": 6.1931573138160925, | |
| "tokens_seen": 18612224 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00028273809523809523, | |
| "loss": 5.0434, | |
| "theoretical_loss": 6.189434736731897, | |
| "tokens_seen": 18677760 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00028373015873015873, | |
| "loss": 5.0132, | |
| "theoretical_loss": 6.185728841158257, | |
| "tokens_seen": 18743296 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00028472222222222223, | |
| "loss": 5.1159, | |
| "theoretical_loss": 6.182039494406309, | |
| "tokens_seen": 18808832 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002857142857142857, | |
| "loss": 5.1786, | |
| "theoretical_loss": 6.178366565300266, | |
| "tokens_seen": 18874368 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002867063492063492, | |
| "loss": 5.1947, | |
| "theoretical_loss": 6.174709924154978, | |
| "tokens_seen": 18939904 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002876984126984127, | |
| "loss": 5.2396, | |
| "theoretical_loss": 6.171069442753909, | |
| "tokens_seen": 19005440 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002886904761904762, | |
| "loss": 5.3947, | |
| "theoretical_loss": 6.1674449943275045, | |
| "tokens_seen": 19070976 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002896825396825397, | |
| "loss": 4.8732, | |
| "theoretical_loss": 6.16383645353196, | |
| "tokens_seen": 19136512 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002906746031746032, | |
| "loss": 5.3178, | |
| "theoretical_loss": 6.160243696428367, | |
| "tokens_seen": 19202048 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002916666666666667, | |
| "loss": 5.1441, | |
| "theoretical_loss": 6.156666600462238, | |
| "tokens_seen": 19267584 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002926587301587302, | |
| "loss": 5.2695, | |
| "theoretical_loss": 6.153105044443393, | |
| "tokens_seen": 19333120 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002936507936507937, | |
| "loss": 5.3181, | |
| "theoretical_loss": 6.149558908526206, | |
| "tokens_seen": 19398656 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029464285714285714, | |
| "loss": 5.2188, | |
| "theoretical_loss": 6.146028074190217, | |
| "tokens_seen": 19464192 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029563492063492064, | |
| "loss": 5.1652, | |
| "theoretical_loss": 6.14251242422106, | |
| "tokens_seen": 19529728 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029662698412698414, | |
| "loss": 5.3475, | |
| "theoretical_loss": 6.139011842691756, | |
| "tokens_seen": 19595264 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 22379, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.661308765411377, | |
| "objective/train/theoretical_loss": 6.135526214944321, | |
| "objective/train/tokens_used": 40120800, | |
| "theoretical_loss": 6.135526214944321, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029761904761904765, | |
| "loss": 5.6218, | |
| "theoretical_loss": 6.135526214944321, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002986111111111111, | |
| "loss": 5.4528, | |
| "theoretical_loss": 6.1320554275717, | |
| "tokens_seen": 19726336 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002996031746031746, | |
| "loss": 5.1971, | |
| "theoretical_loss": 6.128599368400017, | |
| "tokens_seen": 19791872 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003005952380952381, | |
| "loss": 5.2823, | |
| "theoretical_loss": 6.125157926471134, | |
| "tokens_seen": 19857408 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003015873015873016, | |
| "loss": 5.1154, | |
| "theoretical_loss": 6.121730992025516, | |
| "tokens_seen": 19922944 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030257936507936505, | |
| "loss": 4.8293, | |
| "theoretical_loss": 6.118318456485394, | |
| "tokens_seen": 19988480 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030357142857142855, | |
| "loss": 5.0591, | |
| "theoretical_loss": 6.114920212438209, | |
| "tokens_seen": 20054016 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030456349206349205, | |
| "loss": 5.2452, | |
| "theoretical_loss": 6.111536153620355, | |
| "tokens_seen": 20119552 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003055555555555556, | |
| "loss": 5.1158, | |
| "theoretical_loss": 6.108166174901191, | |
| "tokens_seen": 20185088 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030654761904761905, | |
| "loss": 5.0927, | |
| "theoretical_loss": 6.104810172267331, | |
| "tokens_seen": 20250624 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030753968253968255, | |
| "loss": 5.1097, | |
| "theoretical_loss": 6.101468042807199, | |
| "tokens_seen": 20316160 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030853174603174605, | |
| "loss": 5.0109, | |
| "theoretical_loss": 6.098139684695851, | |
| "tokens_seen": 20381696 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030952380952380956, | |
| "loss": 5.2, | |
| "theoretical_loss": 6.094824997180048, | |
| "tokens_seen": 20447232 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000310515873015873, | |
| "loss": 4.919, | |
| "theoretical_loss": 6.091523880563589, | |
| "tokens_seen": 20512768 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003115079365079365, | |
| "loss": 4.7726, | |
| "theoretical_loss": 6.088236236192881, | |
| "tokens_seen": 20578304 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003125, | |
| "loss": 5.1579, | |
| "theoretical_loss": 6.0849619664427586, | |
| "tokens_seen": 20643840 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003134920634920635, | |
| "loss": 5.2649, | |
| "theoretical_loss": 6.0817009747025415, | |
| "tokens_seen": 20709376 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000314484126984127, | |
| "loss": 4.9338, | |
| "theoretical_loss": 6.07845316536232, | |
| "tokens_seen": 20774912 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00031547619047619046, | |
| "loss": 5.2449, | |
| "theoretical_loss": 6.075218443799468, | |
| "tokens_seen": 20840448 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00031646825396825396, | |
| "loss": 5.2222, | |
| "theoretical_loss": 6.071996716365382, | |
| "tokens_seen": 20905984 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00031746031746031746, | |
| "loss": 4.8122, | |
| "theoretical_loss": 6.068787890372443, | |
| "tokens_seen": 20971520 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00031845238095238096, | |
| "loss": 4.983, | |
| "theoretical_loss": 6.0655918740811865, | |
| "tokens_seen": 21037056 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003194444444444444, | |
| "loss": 4.9428, | |
| "theoretical_loss": 6.062408576687682, | |
| "tokens_seen": 21102592 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032043650793650796, | |
| "loss": 5.0493, | |
| "theoretical_loss": 6.059237908311129, | |
| "tokens_seen": 21168128 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032142857142857147, | |
| "loss": 4.8675, | |
| "theoretical_loss": 6.056079779981644, | |
| "tokens_seen": 21233664 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 23601, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.199455738067627, | |
| "objective/train/theoretical_loss": 6.052934103628253, | |
| "objective/train/tokens_used": 41759200, | |
| "theoretical_loss": 6.052934103628253, | |
| "tokens_seen": 21299200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032242063492063497, | |
| "loss": 5.2637, | |
| "theoretical_loss": 6.052934103628253, | |
| "tokens_seen": 21299200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003234126984126984, | |
| "loss": 5.4325, | |
| "theoretical_loss": 6.049800792067078, | |
| "tokens_seen": 21364736 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003244047619047619, | |
| "loss": 4.8274, | |
| "theoretical_loss": 6.046679758989709, | |
| "tokens_seen": 21430272 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003253968253968254, | |
| "loss": 5.2104, | |
| "theoretical_loss": 6.043570918951775, | |
| "tokens_seen": 21495808 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003263888888888889, | |
| "loss": 5.1036, | |
| "theoretical_loss": 6.040474187361681, | |
| "tokens_seen": 21561344 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032738095238095237, | |
| "loss": 4.8811, | |
| "theoretical_loss": 6.037389480469548, | |
| "tokens_seen": 21626880 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032837301587301587, | |
| "loss": 5.0504, | |
| "theoretical_loss": 6.034316715356304, | |
| "tokens_seen": 21692416 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032936507936507937, | |
| "loss": 5.0898, | |
| "theoretical_loss": 6.031255809922974, | |
| "tokens_seen": 21757952 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00033035714285714287, | |
| "loss": 5.0121, | |
| "theoretical_loss": 6.0282066828801195, | |
| "tokens_seen": 21823488 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003313492063492063, | |
| "loss": 5.1363, | |
| "theoretical_loss": 6.0251692537374595, | |
| "tokens_seen": 21889024 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003323412698412698, | |
| "loss": 5.1503, | |
| "theoretical_loss": 6.022143442793643, | |
| "tokens_seen": 21954560 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003333333333333333, | |
| "loss": 5.1066, | |
| "theoretical_loss": 6.0191291711261945, | |
| "tokens_seen": 22020096 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003343253968253968, | |
| "loss": 5.0889, | |
| "theoretical_loss": 6.016126360581609, | |
| "tokens_seen": 22085632 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003353174603174603, | |
| "loss": 4.9077, | |
| "theoretical_loss": 6.013134933765605, | |
| "tokens_seen": 22151168 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003363095238095238, | |
| "loss": 5.2813, | |
| "theoretical_loss": 6.010154814033528, | |
| "tokens_seen": 22216704 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00033730158730158733, | |
| "loss": 4.6999, | |
| "theoretical_loss": 6.007185925480902, | |
| "tokens_seen": 22282240 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00033829365079365083, | |
| "loss": 5.0728, | |
| "theoretical_loss": 6.0042281929341375, | |
| "tokens_seen": 22347776 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00033928571428571433, | |
| "loss": 5.1319, | |
| "theoretical_loss": 6.001281541941363, | |
| "tokens_seen": 22413312 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003402777777777778, | |
| "loss": 5.0915, | |
| "theoretical_loss": 5.998345898763421, | |
| "tokens_seen": 22478848 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003412698412698413, | |
| "loss": 4.8912, | |
| "theoretical_loss": 5.995421190364983, | |
| "tokens_seen": 22544384 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003422619047619048, | |
| "loss": 5.0317, | |
| "theoretical_loss": 5.992507344405814, | |
| "tokens_seen": 22609920 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003432539682539683, | |
| "loss": 5.1603, | |
| "theoretical_loss": 5.9896042892321635, | |
| "tokens_seen": 22675456 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00034424603174603173, | |
| "loss": 4.9822, | |
| "theoretical_loss": 5.986711953868287, | |
| "tokens_seen": 22740992 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00034523809523809523, | |
| "loss": 5.3948, | |
| "theoretical_loss": 5.983830268008107, | |
| "tokens_seen": 22806528 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00034623015873015873, | |
| "loss": 5.222, | |
| "theoretical_loss": 5.980959162006979, | |
| "tokens_seen": 22872064 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 24859, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.439995288848877, | |
| "objective/train/theoretical_loss": 5.978098566873603, | |
| "objective/train/tokens_used": 43397600, | |
| "theoretical_loss": 5.978098566873603, | |
| "tokens_seen": 22937600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00034722222222222224, | |
| "loss": 4.9666, | |
| "theoretical_loss": 5.978098566873603, | |
| "tokens_seen": 22937600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003482142857142857, | |
| "loss": 4.7873, | |
| "theoretical_loss": 5.975248414262053, | |
| "tokens_seen": 23003136 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003492063492063492, | |
| "loss": 5.1429, | |
| "theoretical_loss": 5.972408636463909, | |
| "tokens_seen": 23068672 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003501984126984127, | |
| "loss": 4.8387, | |
| "theoretical_loss": 5.969579166400528, | |
| "tokens_seen": 23134208 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003511904761904762, | |
| "loss": 5.2741, | |
| "theoretical_loss": 5.966759937615427, | |
| "tokens_seen": 23199744 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003521825396825397, | |
| "loss": 4.9993, | |
| "theoretical_loss": 5.96395088426676, | |
| "tokens_seen": 23265280 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003531746031746032, | |
| "loss": 4.838, | |
| "theoretical_loss": 5.961151941119932, | |
| "tokens_seen": 23330816 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003541666666666667, | |
| "loss": 4.8937, | |
| "theoretical_loss": 5.95836304354031, | |
| "tokens_seen": 23396352 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003551587301587302, | |
| "loss": 5.1755, | |
| "theoretical_loss": 5.9555841274860395, | |
| "tokens_seen": 23461888 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003561507936507937, | |
| "loss": 5.0192, | |
| "theoretical_loss": 5.952815129500973, | |
| "tokens_seen": 23527424 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035714285714285714, | |
| "loss": 5.1528, | |
| "theoretical_loss": 5.950055986707699, | |
| "tokens_seen": 23592960 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035813492063492064, | |
| "loss": 5.2473, | |
| "theoretical_loss": 5.9473066368006755, | |
| "tokens_seen": 23658496 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035912698412698415, | |
| "loss": 5.1431, | |
| "theoretical_loss": 5.944567018039454, | |
| "tokens_seen": 23724032 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00036011904761904765, | |
| "loss": 5.2078, | |
| "theoretical_loss": 5.941837069242023, | |
| "tokens_seen": 23789568 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003611111111111111, | |
| "loss": 5.2074, | |
| "theoretical_loss": 5.939116729778224, | |
| "tokens_seen": 23855104 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003621031746031746, | |
| "loss": 5.045, | |
| "theoretical_loss": 5.936405939563279, | |
| "tokens_seen": 23920640 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003630952380952381, | |
| "loss": 5.0272, | |
| "theoretical_loss": 5.93370463905141, | |
| "tokens_seen": 23986176 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003640873015873016, | |
| "loss": 5.0628, | |
| "theoretical_loss": 5.931012769229536, | |
| "tokens_seen": 24051712 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00036507936507936505, | |
| "loss": 4.6127, | |
| "theoretical_loss": 5.928330271611081, | |
| "tokens_seen": 24117248 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00036607142857142855, | |
| "loss": 4.8408, | |
| "theoretical_loss": 5.925657088229862, | |
| "tokens_seen": 24182784 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00036706349206349205, | |
| "loss": 4.9603, | |
| "theoretical_loss": 5.9229931616340545, | |
| "tokens_seen": 24248320 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003680555555555556, | |
| "loss": 5.0127, | |
| "theoretical_loss": 5.920338434880263, | |
| "tokens_seen": 24313856 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00036904761904761905, | |
| "loss": 5.3699, | |
| "theoretical_loss": 5.9176928515276535, | |
| "tokens_seen": 24379392 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037003968253968255, | |
| "loss": 4.9974, | |
| "theoretical_loss": 5.915056355632197, | |
| "tokens_seen": 24444928 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037103174603174606, | |
| "loss": 4.9348, | |
| "theoretical_loss": 5.912428891740967, | |
| "tokens_seen": 24510464 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 25411, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.414747714996338, | |
| "objective/train/theoretical_loss": 5.90981040488653, | |
| "objective/train/tokens_used": 45036000, | |
| "theoretical_loss": 5.90981040488653, | |
| "tokens_seen": 24576000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037202380952380956, | |
| "loss": 5.0376, | |
| "theoretical_loss": 5.90981040488653, | |
| "tokens_seen": 24576000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000373015873015873, | |
| "loss": 5.0724, | |
| "theoretical_loss": 5.907200840581417, | |
| "tokens_seen": 24641536 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003740079365079365, | |
| "loss": 5.2247, | |
| "theoretical_loss": 5.904600144812672, | |
| "tokens_seen": 24707072 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000375, | |
| "loss": 5.4811, | |
| "theoretical_loss": 5.902008264036468, | |
| "tokens_seen": 24772608 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003759920634920635, | |
| "loss": 4.9107, | |
| "theoretical_loss": 5.899425145172803, | |
| "tokens_seen": 24838144 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000376984126984127, | |
| "loss": 5.0933, | |
| "theoretical_loss": 5.896850735600281, | |
| "tokens_seen": 24903680 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037797619047619046, | |
| "loss": 5.1802, | |
| "theoretical_loss": 5.8942849831509445, | |
| "tokens_seen": 24969216 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037896825396825396, | |
| "loss": 5.2521, | |
| "theoretical_loss": 5.891727836105194, | |
| "tokens_seen": 25034752 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037996031746031746, | |
| "loss": 4.8607, | |
| "theoretical_loss": 5.889179243186776, | |
| "tokens_seen": 25100288 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00038095238095238096, | |
| "loss": 5.0973, | |
| "theoretical_loss": 5.886639153557828, | |
| "tokens_seen": 25165824 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003819444444444444, | |
| "loss": 5.3397, | |
| "theoretical_loss": 5.8841075168140105, | |
| "tokens_seen": 25231360 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00038293650793650797, | |
| "loss": 4.7777, | |
| "theoretical_loss": 5.88158428297969, | |
| "tokens_seen": 25296896 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00038392857142857147, | |
| "loss": 4.7219, | |
| "theoretical_loss": 5.879069402503189, | |
| "tokens_seen": 25362432 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00038492063492063497, | |
| "loss": 5.0299, | |
| "theoretical_loss": 5.876562826252119, | |
| "tokens_seen": 25427968 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003859126984126984, | |
| "loss": 4.8369, | |
| "theoretical_loss": 5.874064505508748, | |
| "tokens_seen": 25493504 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003869047619047619, | |
| "loss": 5.0017, | |
| "theoretical_loss": 5.871574391965453, | |
| "tokens_seen": 25559040 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003878968253968254, | |
| "loss": 4.9705, | |
| "theoretical_loss": 5.8690924377202265, | |
| "tokens_seen": 25624576 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003888888888888889, | |
| "loss": 5.034, | |
| "theoretical_loss": 5.866618595272241, | |
| "tokens_seen": 25690112 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00038988095238095237, | |
| "loss": 4.7181, | |
| "theoretical_loss": 5.864152817517482, | |
| "tokens_seen": 25755648 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00039087301587301587, | |
| "loss": 4.9562, | |
| "theoretical_loss": 5.8616950577444245, | |
| "tokens_seen": 25821184 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00039186507936507937, | |
| "loss": 4.9405, | |
| "theoretical_loss": 5.859245269629783, | |
| "tokens_seen": 25886720 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003928571428571429, | |
| "loss": 5.099, | |
| "theoretical_loss": 5.85680340723431, | |
| "tokens_seen": 25952256 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003938492063492063, | |
| "loss": 4.6503, | |
| "theoretical_loss": 5.85436942499865, | |
| "tokens_seen": 26017792 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003948412698412698, | |
| "loss": 4.8672, | |
| "theoretical_loss": 5.851943277739253, | |
| "tokens_seen": 26083328 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003958333333333333, | |
| "loss": 5.0201, | |
| "theoretical_loss": 5.84952492064434, | |
| "tokens_seen": 26148864 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 26668, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.337137222290039, | |
| "objective/train/theoretical_loss": 5.847114309269919, | |
| "objective/train/tokens_used": 46674400, | |
| "theoretical_loss": 5.847114309269919, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003968253968253968, | |
| "loss": 5.1138, | |
| "theoretical_loss": 5.847114309269919, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003978174603174603, | |
| "loss": 4.8239, | |
| "theoretical_loss": 5.844711399535855, | |
| "tokens_seen": 26279936 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00039880952380952383, | |
| "loss": 4.7304, | |
| "theoretical_loss": 5.842316147722, | |
| "tokens_seen": 26345472 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00039980158730158733, | |
| "loss": 5.0092, | |
| "theoretical_loss": 5.839928510464356, | |
| "tokens_seen": 26411008 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040079365079365083, | |
| "loss": 4.4343, | |
| "theoretical_loss": 5.837548444751306, | |
| "tokens_seen": 26476544 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040178571428571433, | |
| "loss": 4.9309, | |
| "theoretical_loss": 5.835175907919885, | |
| "tokens_seen": 26542080 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004027777777777778, | |
| "loss": 5.0148, | |
| "theoretical_loss": 5.832810857652097, | |
| "tokens_seen": 26607616 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004037698412698413, | |
| "loss": 4.9709, | |
| "theoretical_loss": 5.830453251971296, | |
| "tokens_seen": 26673152 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004047619047619048, | |
| "loss": 5.1225, | |
| "theoretical_loss": 5.8281030492385835, | |
| "tokens_seen": 26738688 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004057539682539683, | |
| "loss": 4.9439, | |
| "theoretical_loss": 5.82576020814929, | |
| "tokens_seen": 26804224 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040674603174603173, | |
| "loss": 5.154, | |
| "theoretical_loss": 5.823424687729469, | |
| "tokens_seen": 26869760 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040773809523809523, | |
| "loss": 4.6484, | |
| "theoretical_loss": 5.821096447332456, | |
| "tokens_seen": 26935296 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040873015873015874, | |
| "loss": 4.6262, | |
| "theoretical_loss": 5.818775446635469, | |
| "tokens_seen": 27000832 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040972222222222224, | |
| "loss": 5.0236, | |
| "theoretical_loss": 5.816461645636238, | |
| "tokens_seen": 27066368 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004107142857142857, | |
| "loss": 5.1234, | |
| "theoretical_loss": 5.814155004649702, | |
| "tokens_seen": 27131904 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004117063492063492, | |
| "loss": 4.6509, | |
| "theoretical_loss": 5.811855484304724, | |
| "tokens_seen": 27197440 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004126984126984127, | |
| "loss": 4.9814, | |
| "theoretical_loss": 5.809563045540864, | |
| "tokens_seen": 27262976 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004136904761904762, | |
| "loss": 5.0187, | |
| "theoretical_loss": 5.8072776496051866, | |
| "tokens_seen": 27328512 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004146825396825397, | |
| "loss": 4.9255, | |
| "theoretical_loss": 5.804999258049106, | |
| "tokens_seen": 27394048 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004156746031746032, | |
| "loss": 5.0265, | |
| "theoretical_loss": 5.802727832725283, | |
| "tokens_seen": 27459584 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004166666666666667, | |
| "loss": 4.8827, | |
| "theoretical_loss": 5.800463335784541, | |
| "tokens_seen": 27525120 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004176587301587302, | |
| "loss": 5.0962, | |
| "theoretical_loss": 5.798205729672842, | |
| "tokens_seen": 27590656 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004186507936507937, | |
| "loss": 5.0409, | |
| "theoretical_loss": 5.795954977128286, | |
| "tokens_seen": 27656192 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00041964285714285714, | |
| "loss": 4.8332, | |
| "theoretical_loss": 5.7937110411781525, | |
| "tokens_seen": 27721728 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042063492063492065, | |
| "loss": 4.813, | |
| "theoretical_loss": 5.79147388513598, | |
| "tokens_seen": 27787264 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 27295, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.60736083984375, | |
| "objective/train/theoretical_loss": 5.789243472598683, | |
| "objective/train/tokens_used": 48312800, | |
| "theoretical_loss": 5.789243472598683, | |
| "tokens_seen": 27852800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042162698412698415, | |
| "loss": 4.735, | |
| "theoretical_loss": 5.789243472598683, | |
| "tokens_seen": 27852800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042261904761904765, | |
| "loss": 5.0103, | |
| "theoretical_loss": 5.787019767443696, | |
| "tokens_seen": 27918336 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004236111111111111, | |
| "loss": 4.9487, | |
| "theoretical_loss": 5.784802733826166, | |
| "tokens_seen": 27983872 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004246031746031746, | |
| "loss": 5.0393, | |
| "theoretical_loss": 5.782592336176171, | |
| "tokens_seen": 28049408 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004255952380952381, | |
| "loss": 4.8239, | |
| "theoretical_loss": 5.780388539195972, | |
| "tokens_seen": 28114944 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004265873015873016, | |
| "loss": 4.8301, | |
| "theoretical_loss": 5.778191307857307, | |
| "tokens_seen": 28180480 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042757936507936505, | |
| "loss": 4.8818, | |
| "theoretical_loss": 5.776000607398707, | |
| "tokens_seen": 28246016 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042857142857142855, | |
| "loss": 5.0365, | |
| "theoretical_loss": 5.773816403322854, | |
| "tokens_seen": 28311552 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042956349206349205, | |
| "loss": 4.8806, | |
| "theoretical_loss": 5.7716386613939665, | |
| "tokens_seen": 28377088 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004305555555555556, | |
| "loss": 4.8212, | |
| "theoretical_loss": 5.769467347635221, | |
| "tokens_seen": 28442624 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00043154761904761905, | |
| "loss": 4.6856, | |
| "theoretical_loss": 5.767302428326196, | |
| "tokens_seen": 28508160 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00043253968253968256, | |
| "loss": 5.1099, | |
| "theoretical_loss": 5.765143870000358, | |
| "tokens_seen": 28573696 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00043353174603174606, | |
| "loss": 4.9942, | |
| "theoretical_loss": 5.762991639442574, | |
| "tokens_seen": 28639232 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00043452380952380956, | |
| "loss": 4.7269, | |
| "theoretical_loss": 5.7608457036866465, | |
| "tokens_seen": 28704768 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000435515873015873, | |
| "loss": 4.7079, | |
| "theoretical_loss": 5.758706030012889, | |
| "tokens_seen": 28770304 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004365079365079365, | |
| "loss": 5.0535, | |
| "theoretical_loss": 5.756572585945728, | |
| "tokens_seen": 28835840 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004375, | |
| "loss": 4.9953, | |
| "theoretical_loss": 5.754445339251326, | |
| "tokens_seen": 28901376 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004384920634920635, | |
| "loss": 4.8239, | |
| "theoretical_loss": 5.752324257935244, | |
| "tokens_seen": 28966912 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000439484126984127, | |
| "loss": 4.8738, | |
| "theoretical_loss": 5.750209310240125, | |
| "tokens_seen": 29032448 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044047619047619046, | |
| "loss": 4.5774, | |
| "theoretical_loss": 5.7481004646434055, | |
| "tokens_seen": 29097984 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044146825396825396, | |
| "loss": 4.7821, | |
| "theoretical_loss": 5.745997689855058, | |
| "tokens_seen": 29163520 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044246031746031746, | |
| "loss": 4.9197, | |
| "theoretical_loss": 5.743900954815356, | |
| "tokens_seen": 29229056 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044345238095238096, | |
| "loss": 4.9178, | |
| "theoretical_loss": 5.741810228692663, | |
| "tokens_seen": 29294592 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004444444444444444, | |
| "loss": 4.9811, | |
| "theoretical_loss": 5.739725480881262, | |
| "tokens_seen": 29360128 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044543650793650797, | |
| "loss": 4.595, | |
| "theoretical_loss": 5.737646680999193, | |
| "tokens_seen": 29425664 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 28553, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.193212032318115, | |
| "objective/train/theoretical_loss": 5.73557379888612, | |
| "objective/train/tokens_used": 49951200, | |
| "theoretical_loss": 5.73557379888612, | |
| "tokens_seen": 29491200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044642857142857147, | |
| "loss": 4.6289, | |
| "theoretical_loss": 5.73557379888612, | |
| "tokens_seen": 29491200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044742063492063497, | |
| "loss": 4.5175, | |
| "theoretical_loss": 5.733506804601236, | |
| "tokens_seen": 29556736 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004484126984126984, | |
| "loss": 4.97, | |
| "theoretical_loss": 5.7314456684211725, | |
| "tokens_seen": 29622272 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004494047619047619, | |
| "loss": 4.9007, | |
| "theoretical_loss": 5.729390360837952, | |
| "tokens_seen": 29687808 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004503968253968254, | |
| "loss": 5.0718, | |
| "theoretical_loss": 5.7273408525569485, | |
| "tokens_seen": 29753344 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004513888888888889, | |
| "loss": 4.8076, | |
| "theoretical_loss": 5.725297114494884, | |
| "tokens_seen": 29818880 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00045238095238095237, | |
| "loss": 4.9151, | |
| "theoretical_loss": 5.7232591177778405, | |
| "tokens_seen": 29884416 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00045337301587301587, | |
| "loss": 4.7611, | |
| "theoretical_loss": 5.7212268337393, | |
| "tokens_seen": 29949952 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00045436507936507937, | |
| "loss": 4.6533, | |
| "theoretical_loss": 5.719200233918203, | |
| "tokens_seen": 30015488 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004553571428571429, | |
| "loss": 4.7689, | |
| "theoretical_loss": 5.717179290057032, | |
| "tokens_seen": 30081024 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004563492063492063, | |
| "loss": 4.753, | |
| "theoretical_loss": 5.715163974099917, | |
| "tokens_seen": 30146560 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004573412698412698, | |
| "loss": 4.9783, | |
| "theoretical_loss": 5.713154258190757, | |
| "tokens_seen": 30212096 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004583333333333333, | |
| "loss": 4.8919, | |
| "theoretical_loss": 5.711150114671375, | |
| "tokens_seen": 30277632 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004593253968253968, | |
| "loss": 4.6246, | |
| "theoretical_loss": 5.709151516079683, | |
| "tokens_seen": 30343168 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046031746031746033, | |
| "loss": 4.8047, | |
| "theoretical_loss": 5.707158435147875, | |
| "tokens_seen": 30408704 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046130952380952383, | |
| "loss": 4.7619, | |
| "theoretical_loss": 5.705170844800628, | |
| "tokens_seen": 30474240 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046230158730158733, | |
| "loss": 5.0389, | |
| "theoretical_loss": 5.703188718153347, | |
| "tokens_seen": 30539776 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046329365079365083, | |
| "loss": 4.7978, | |
| "theoretical_loss": 5.7012120285104055, | |
| "tokens_seen": 30605312 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046428571428571433, | |
| "loss": 4.8182, | |
| "theoretical_loss": 5.699240749363417, | |
| "tokens_seen": 30670848 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004652777777777778, | |
| "loss": 5.1046, | |
| "theoretical_loss": 5.69727485438953, | |
| "tokens_seen": 30736384 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004662698412698413, | |
| "loss": 4.9301, | |
| "theoretical_loss": 5.695314317449732, | |
| "tokens_seen": 30801920 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004672619047619048, | |
| "loss": 4.9513, | |
| "theoretical_loss": 5.69335911258718, | |
| "tokens_seen": 30867456 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004682539682539683, | |
| "loss": 4.7506, | |
| "theoretical_loss": 5.691409214025544, | |
| "tokens_seen": 30932992 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046924603174603173, | |
| "loss": 4.9203, | |
| "theoretical_loss": 5.689464596167383, | |
| "tokens_seen": 30998528 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00047023809523809523, | |
| "loss": 4.6588, | |
| "theoretical_loss": 5.687525233592513, | |
| "tokens_seen": 31064064 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 29210, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.980744361877441, | |
| "objective/train/theoretical_loss": 5.685591101056422, | |
| "objective/train/tokens_used": 51589600, | |
| "theoretical_loss": 5.685591101056422, | |
| "tokens_seen": 31129600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00047123015873015874, | |
| "loss": 4.8131, | |
| "theoretical_loss": 5.685591101056422, | |
| "tokens_seen": 31129600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00047222222222222224, | |
| "loss": 4.9729, | |
| "theoretical_loss": 5.683662173488678, | |
| "tokens_seen": 31195136 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004732142857142857, | |
| "loss": 4.9988, | |
| "theoretical_loss": 5.681738425991377, | |
| "tokens_seen": 31260672 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004742063492063492, | |
| "loss": 4.9296, | |
| "theoretical_loss": 5.679819833837586, | |
| "tokens_seen": 31326208 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004751984126984127, | |
| "loss": 4.8866, | |
| "theoretical_loss": 5.677906372469826, | |
| "tokens_seen": 31391744 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004761904761904762, | |
| "loss": 4.8619, | |
| "theoretical_loss": 5.675998017498549, | |
| "tokens_seen": 31457280 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004771825396825397, | |
| "loss": 4.4881, | |
| "theoretical_loss": 5.674094744700648, | |
| "tokens_seen": 31522816 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004781746031746032, | |
| "loss": 4.7385, | |
| "theoretical_loss": 5.672196530017979, | |
| "tokens_seen": 31588352 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004791666666666667, | |
| "loss": 4.9446, | |
| "theoretical_loss": 5.670303349555893, | |
| "tokens_seen": 31653888 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004801587301587302, | |
| "loss": 4.5805, | |
| "theoretical_loss": 5.668415179581795, | |
| "tokens_seen": 31719424 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004811507936507937, | |
| "loss": 4.943, | |
| "theoretical_loss": 5.666531996523711, | |
| "tokens_seen": 31784960 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00048214285714285715, | |
| "loss": 4.9994, | |
| "theoretical_loss": 5.664653776968867, | |
| "tokens_seen": 31850496 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00048313492063492065, | |
| "loss": 4.9432, | |
| "theoretical_loss": 5.6627804976622995, | |
| "tokens_seen": 31916032 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00048412698412698415, | |
| "loss": 4.4612, | |
| "theoretical_loss": 5.660912135505461, | |
| "tokens_seen": 31981568 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00048511904761904765, | |
| "loss": 5.0296, | |
| "theoretical_loss": 5.659048667554853, | |
| "tokens_seen": 32047104 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004861111111111111, | |
| "loss": 4.7342, | |
| "theoretical_loss": 5.657190071020672, | |
| "tokens_seen": 32112640 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004871031746031746, | |
| "loss": 5.0271, | |
| "theoretical_loss": 5.65533632326547, | |
| "tokens_seen": 32178176 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004880952380952381, | |
| "loss": 4.5208, | |
| "theoretical_loss": 5.6534874018028205, | |
| "tokens_seen": 32243712 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004890873015873016, | |
| "loss": 4.8883, | |
| "theoretical_loss": 5.651643284296018, | |
| "tokens_seen": 32309248 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004900793650793651, | |
| "loss": 4.5918, | |
| "theoretical_loss": 5.6498039485567695, | |
| "tokens_seen": 32374784 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004910714285714286, | |
| "loss": 4.8362, | |
| "theoretical_loss": 5.6479693725439155, | |
| "tokens_seen": 32440320 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000492063492063492, | |
| "loss": 4.4816, | |
| "theoretical_loss": 5.646139534362161, | |
| "tokens_seen": 32505856 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004930555555555556, | |
| "loss": 4.7304, | |
| "theoretical_loss": 5.644314412260815, | |
| "tokens_seen": 32571392 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004940476190476191, | |
| "loss": 4.7639, | |
| "theoretical_loss": 5.642493984632544, | |
| "tokens_seen": 32636928 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004950396825396826, | |
| "loss": 4.486, | |
| "theoretical_loss": 5.640678230012151, | |
| "tokens_seen": 32702464 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 30192, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.205135822296143, | |
| "objective/train/theoretical_loss": 5.638867127075349, | |
| "objective/train/tokens_used": 53228000, | |
| "theoretical_loss": 5.638867127075349, | |
| "tokens_seen": 32768000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000496031746031746, | |
| "loss": 4.6383, | |
| "theoretical_loss": 5.638867127075349, | |
| "tokens_seen": 32768000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004970238095238095, | |
| "loss": 4.7665, | |
| "theoretical_loss": 5.63706065463756, | |
| "tokens_seen": 32833536 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498015873015873, | |
| "loss": 4.6651, | |
| "theoretical_loss": 5.635258791652723, | |
| "tokens_seen": 32899072 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004990079365079365, | |
| "loss": 4.6032, | |
| "theoretical_loss": 5.633461517212115, | |
| "tokens_seen": 32964608 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0005, | |
| "loss": 4.8611, | |
| "theoretical_loss": 5.631668810543181, | |
| "tokens_seen": 33030144 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004999899699097292, | |
| "loss": 4.4403, | |
| "theoretical_loss": 5.629880651008384, | |
| "tokens_seen": 33095680 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004999799398194584, | |
| "loss": 4.6342, | |
| "theoretical_loss": 5.628097018104057, | |
| "tokens_seen": 33161216 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004999699097291876, | |
| "loss": 4.789, | |
| "theoretical_loss": 5.626317891459275, | |
| "tokens_seen": 33226752 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004999598796389167, | |
| "loss": 4.4669, | |
| "theoretical_loss": 5.6245432508347335, | |
| "tokens_seen": 33292288 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499949849548646, | |
| "loss": 4.5265, | |
| "theoretical_loss": 5.622773076121648, | |
| "tokens_seen": 33357824 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004999398194583751, | |
| "loss": 4.8668, | |
| "theoretical_loss": 5.621007347340646, | |
| "tokens_seen": 33423360 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004999297893681044, | |
| "loss": 4.663, | |
| "theoretical_loss": 5.619246044640693, | |
| "tokens_seen": 33488896 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004999197592778335, | |
| "loss": 4.51, | |
| "theoretical_loss": 5.617489148298015, | |
| "tokens_seen": 33554432 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004999097291875627, | |
| "loss": 4.4827, | |
| "theoretical_loss": 5.615736638715033, | |
| "tokens_seen": 33619968 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004998996990972919, | |
| "loss": 4.6552, | |
| "theoretical_loss": 5.613988496419315, | |
| "tokens_seen": 33685504 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004998896690070211, | |
| "loss": 4.5444, | |
| "theoretical_loss": 5.6122447020625374, | |
| "tokens_seen": 33751040 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004998796389167503, | |
| "loss": 4.7462, | |
| "theoretical_loss": 5.610505236419448, | |
| "tokens_seen": 33816576 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004998696088264795, | |
| "loss": 4.6343, | |
| "theoretical_loss": 5.608770080386854, | |
| "tokens_seen": 33882112 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004998595787362087, | |
| "loss": 4.3194, | |
| "theoretical_loss": 5.6070392149826045, | |
| "tokens_seen": 33947648 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004998495486459378, | |
| "loss": 4.7376, | |
| "theoretical_loss": 5.605312621344602, | |
| "tokens_seen": 34013184 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499839518555667, | |
| "loss": 4.6319, | |
| "theoretical_loss": 5.603590280729803, | |
| "tokens_seen": 34078720 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004998294884653962, | |
| "loss": 4.3936, | |
| "theoretical_loss": 5.601872174513245, | |
| "tokens_seen": 34144256 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004998194583751254, | |
| "loss": 4.8045, | |
| "theoretical_loss": 5.60015828418708, | |
| "tokens_seen": 34209792 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004998094282848546, | |
| "loss": 4.8346, | |
| "theoretical_loss": 5.59844859135961, | |
| "tokens_seen": 34275328 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004997993981945837, | |
| "loss": 4.4444, | |
| "theoretical_loss": 5.596743077754337, | |
| "tokens_seen": 34340864 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 30758, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.619051933288574, | |
| "objective/train/theoretical_loss": 5.5950417252090325, | |
| "objective/train/tokens_used": 54866400, | |
| "theoretical_loss": 5.5950417252090325, | |
| "tokens_seen": 34406400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004997893681043129, | |
| "loss": 4.5684, | |
| "theoretical_loss": 5.5950417252090325, | |
| "tokens_seen": 34406400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004997793380140421, | |
| "loss": 4.7403, | |
| "theoretical_loss": 5.593344515674795, | |
| "tokens_seen": 34471936 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004997693079237714, | |
| "loss": 4.7682, | |
| "theoretical_loss": 5.591651431215139, | |
| "tokens_seen": 34537472 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004997592778335005, | |
| "loss": 4.9194, | |
| "theoretical_loss": 5.589962454005072, | |
| "tokens_seen": 34603008 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004997492477432298, | |
| "loss": 4.7926, | |
| "theoretical_loss": 5.588277566330207, | |
| "tokens_seen": 34668544 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004997392176529588, | |
| "loss": 4.5919, | |
| "theoretical_loss": 5.586596750585855, | |
| "tokens_seen": 34734080 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004997291875626881, | |
| "loss": 4.8471, | |
| "theoretical_loss": 5.584919989276146, | |
| "tokens_seen": 34799616 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004997191574724173, | |
| "loss": 4.8823, | |
| "theoretical_loss": 5.583247265013155, | |
| "tokens_seen": 34865152 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004997091273821465, | |
| "loss": 4.7952, | |
| "theoretical_loss": 5.581578560516027, | |
| "tokens_seen": 34930688 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004996990972918757, | |
| "loss": 4.8327, | |
| "theoretical_loss": 5.57991385861013, | |
| "tokens_seen": 34996224 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004996890672016048, | |
| "loss": 4.3745, | |
| "theoretical_loss": 5.578253142226191, | |
| "tokens_seen": 35061760 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499679037111334, | |
| "loss": 4.6921, | |
| "theoretical_loss": 5.576596394399467, | |
| "tokens_seen": 35127296 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004996690070210632, | |
| "loss": 4.6698, | |
| "theoretical_loss": 5.574943598268902, | |
| "tokens_seen": 35192832 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004996589769307924, | |
| "loss": 4.9374, | |
| "theoretical_loss": 5.573294737076306, | |
| "tokens_seen": 35258368 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004996489468405216, | |
| "loss": 4.5814, | |
| "theoretical_loss": 5.571649794165537, | |
| "tokens_seen": 35323904 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004996389167502507, | |
| "loss": 4.6064, | |
| "theoretical_loss": 5.570008752981688, | |
| "tokens_seen": 35389440 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004996288866599799, | |
| "loss": 4.7137, | |
| "theoretical_loss": 5.5683715970702945, | |
| "tokens_seen": 35454976 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004996188565697091, | |
| "loss": 4.7919, | |
| "theoretical_loss": 5.566738310076531, | |
| "tokens_seen": 35520512 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004996088264794383, | |
| "loss": 4.5989, | |
| "theoretical_loss": 5.5651088757444285, | |
| "tokens_seen": 35586048 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004995987963891675, | |
| "loss": 4.4273, | |
| "theoretical_loss": 5.5634832779161, | |
| "tokens_seen": 35651584 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004995887662988968, | |
| "loss": 4.6147, | |
| "theoretical_loss": 5.561861500530965, | |
| "tokens_seen": 35717120 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004995787362086258, | |
| "loss": 4.5803, | |
| "theoretical_loss": 5.560243527624989, | |
| "tokens_seen": 35782656 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004995687061183551, | |
| "loss": 4.762, | |
| "theoretical_loss": 5.558629343329924, | |
| "tokens_seen": 35848192 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004995586760280842, | |
| "loss": 4.7117, | |
| "theoretical_loss": 5.557018931872566, | |
| "tokens_seen": 35913728 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004995486459378135, | |
| "loss": 4.7915, | |
| "theoretical_loss": 5.555412277574007, | |
| "tokens_seen": 35979264 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 31707, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.547565460205078, | |
| "objective/train/theoretical_loss": 5.553809364848902, | |
| "objective/train/tokens_used": 56504800, | |
| "theoretical_loss": 5.553809364848902, | |
| "tokens_seen": 36044800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004995386158475427, | |
| "loss": 4.6425, | |
| "theoretical_loss": 5.553809364848902, | |
| "tokens_seen": 36044800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004995285857572718, | |
| "loss": 4.5963, | |
| "theoretical_loss": 5.552210178204751, | |
| "tokens_seen": 36110336 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499518555667001, | |
| "loss": 4.6793, | |
| "theoretical_loss": 5.550614702241162, | |
| "tokens_seen": 36175872 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004995085255767302, | |
| "loss": 4.7561, | |
| "theoretical_loss": 5.549022921649151, | |
| "tokens_seen": 36241408 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004994984954864594, | |
| "loss": 4.6607, | |
| "theoretical_loss": 5.54743482121043, | |
| "tokens_seen": 36306944 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004994884653961886, | |
| "loss": 4.6252, | |
| "theoretical_loss": 5.545850385796712, | |
| "tokens_seen": 36372480 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004994784353059178, | |
| "loss": 4.4331, | |
| "theoretical_loss": 5.544269600369009, | |
| "tokens_seen": 36438016 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004994684052156469, | |
| "loss": 4.643, | |
| "theoretical_loss": 5.5426924499769505, | |
| "tokens_seen": 36503552 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004994583751253761, | |
| "loss": 4.7548, | |
| "theoretical_loss": 5.541118919758107, | |
| "tokens_seen": 36569088 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004994483450351053, | |
| "loss": 4.7337, | |
| "theoretical_loss": 5.539548994937308, | |
| "tokens_seen": 36634624 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004994383149448345, | |
| "loss": 4.3641, | |
| "theoretical_loss": 5.537982660825981, | |
| "tokens_seen": 36700160 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004994282848545637, | |
| "loss": 4.5599, | |
| "theoretical_loss": 5.536419902821491, | |
| "tokens_seen": 36765696 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004994182547642928, | |
| "loss": 4.6875, | |
| "theoretical_loss": 5.534860706406477, | |
| "tokens_seen": 36831232 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004994082246740221, | |
| "loss": 4.7352, | |
| "theoretical_loss": 5.533305057148215, | |
| "tokens_seen": 36896768 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004993981945837512, | |
| "loss": 4.6149, | |
| "theoretical_loss": 5.531752940697968, | |
| "tokens_seen": 36962304 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004993881644934805, | |
| "loss": 4.5067, | |
| "theoretical_loss": 5.5302043427903484, | |
| "tokens_seen": 37027840 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004993781344032096, | |
| "loss": 4.9474, | |
| "theoretical_loss": 5.528659249242693, | |
| "tokens_seen": 37093376 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004993681043129389, | |
| "loss": 4.4017, | |
| "theoretical_loss": 5.527117645954435, | |
| "tokens_seen": 37158912 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499358074222668, | |
| "loss": 4.5323, | |
| "theoretical_loss": 5.525579518906484, | |
| "tokens_seen": 37224448 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004993480441323972, | |
| "loss": 4.4802, | |
| "theoretical_loss": 5.524044854160613, | |
| "tokens_seen": 37289984 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004993380140421264, | |
| "loss": 4.4531, | |
| "theoretical_loss": 5.522513637858855, | |
| "tokens_seen": 37355520 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004993279839518556, | |
| "loss": 4.6216, | |
| "theoretical_loss": 5.520985856222893, | |
| "tokens_seen": 37421056 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004993179538615848, | |
| "loss": 4.5407, | |
| "theoretical_loss": 5.519461495553473, | |
| "tokens_seen": 37486592 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004993079237713139, | |
| "loss": 4.4097, | |
| "theoretical_loss": 5.5179405422298045, | |
| "tokens_seen": 37552128 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004992978936810431, | |
| "loss": 4.406, | |
| "theoretical_loss": 5.516422982708983, | |
| "tokens_seen": 37617664 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 32203, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.049692630767822, | |
| "objective/train/theoretical_loss": 5.514908803525402, | |
| "objective/train/tokens_used": 58143200, | |
| "theoretical_loss": 5.514908803525402, | |
| "tokens_seen": 37683200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004992878635907723, | |
| "loss": 4.3638, | |
| "theoretical_loss": 5.514908803525402, | |
| "tokens_seen": 37683200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004992778335005015, | |
| "loss": 4.6329, | |
| "theoretical_loss": 5.513397991290185, | |
| "tokens_seen": 37748736 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004992678034102307, | |
| "loss": 4.6538, | |
| "theoretical_loss": 5.511890532690611, | |
| "tokens_seen": 37814272 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004992577733199598, | |
| "loss": 4.7513, | |
| "theoretical_loss": 5.510386414489556, | |
| "tokens_seen": 37879808 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499247743229689, | |
| "loss": 4.6591, | |
| "theoretical_loss": 5.5088856235249235, | |
| "tokens_seen": 37945344 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004992377131394183, | |
| "loss": 4.4618, | |
| "theoretical_loss": 5.507388146709105, | |
| "tokens_seen": 38010880 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004992276830491475, | |
| "loss": 4.4231, | |
| "theoretical_loss": 5.505893971028418, | |
| "tokens_seen": 38076416 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004992176529588767, | |
| "loss": 4.2713, | |
| "theoretical_loss": 5.504403083542568, | |
| "tokens_seen": 38141952 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004992076228686059, | |
| "loss": 4.4092, | |
| "theoretical_loss": 5.502915471384111, | |
| "tokens_seen": 38207488 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499197592778335, | |
| "loss": 4.7235, | |
| "theoretical_loss": 5.501431121757914, | |
| "tokens_seen": 38273024 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991875626880642, | |
| "loss": 4.3762, | |
| "theoretical_loss": 5.49995002194063, | |
| "tokens_seen": 38338560 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991775325977934, | |
| "loss": 4.6923, | |
| "theoretical_loss": 5.498472159280171, | |
| "tokens_seen": 38404096 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991675025075226, | |
| "loss": 4.8392, | |
| "theoretical_loss": 5.4969975211951905, | |
| "tokens_seen": 38469632 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991574724172518, | |
| "loss": 4.5345, | |
| "theoretical_loss": 5.495526095174563, | |
| "tokens_seen": 38535168 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499147442326981, | |
| "loss": 4.6407, | |
| "theoretical_loss": 5.494057868776883, | |
| "tokens_seen": 38600704 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991374122367101, | |
| "loss": 4.5119, | |
| "theoretical_loss": 5.492592829629946, | |
| "tokens_seen": 38666240 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991273821464393, | |
| "loss": 4.6456, | |
| "theoretical_loss": 5.491130965430262, | |
| "tokens_seen": 38731776 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991173520561685, | |
| "loss": 4.2981, | |
| "theoretical_loss": 5.489672263942543, | |
| "tokens_seen": 38797312 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991073219658977, | |
| "loss": 4.7488, | |
| "theoretical_loss": 5.488216712999222, | |
| "tokens_seen": 38862848 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499097291875627, | |
| "loss": 4.5654, | |
| "theoretical_loss": 5.486764300499962, | |
| "tokens_seen": 38928384 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499087261785356, | |
| "loss": 4.3462, | |
| "theoretical_loss": 5.4853150144111655, | |
| "tokens_seen": 38993920 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004990772316950853, | |
| "loss": 4.6389, | |
| "theoretical_loss": 5.4838688427655065, | |
| "tokens_seen": 39059456 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004990672016048144, | |
| "loss": 4.5166, | |
| "theoretical_loss": 5.482425773661445, | |
| "tokens_seen": 39124992 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004990571715145437, | |
| "loss": 4.5858, | |
| "theoretical_loss": 5.480985795262759, | |
| "tokens_seen": 39190528 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004990471414242729, | |
| "loss": 4.6098, | |
| "theoretical_loss": 5.479548895798082, | |
| "tokens_seen": 39256064 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 33134, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.761120319366455, | |
| "objective/train/theoretical_loss": 5.478115063560433, | |
| "objective/train/tokens_used": 59781600, | |
| "theoretical_loss": 5.478115063560433, | |
| "tokens_seen": 39321600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499037111334002, | |
| "loss": 4.4957, | |
| "theoretical_loss": 5.478115063560433, | |
| "tokens_seen": 39321600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004990270812437312, | |
| "loss": 4.7046, | |
| "theoretical_loss": 5.476684286906764, | |
| "tokens_seen": 39387136 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004990170511534604, | |
| "loss": 4.2948, | |
| "theoretical_loss": 5.475256554257497, | |
| "tokens_seen": 39452672 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004990070210631896, | |
| "loss": 4.5969, | |
| "theoretical_loss": 5.473831854096085, | |
| "tokens_seen": 39518208 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004989969909729188, | |
| "loss": 4.5511, | |
| "theoretical_loss": 5.472410174968553, | |
| "tokens_seen": 39583744 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498986960882648, | |
| "loss": 4.4106, | |
| "theoretical_loss": 5.470991505483065, | |
| "tokens_seen": 39649280 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004989769307923771, | |
| "loss": 4.623, | |
| "theoretical_loss": 5.469575834309477, | |
| "tokens_seen": 39714816 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004989669007021063, | |
| "loss": 4.5077, | |
| "theoretical_loss": 5.468163150178907, | |
| "tokens_seen": 39780352 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004989568706118355, | |
| "loss": 4.475, | |
| "theoretical_loss": 5.466753441883302, | |
| "tokens_seen": 39845888 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004989468405215647, | |
| "loss": 4.6451, | |
| "theoretical_loss": 5.465346698275008, | |
| "tokens_seen": 39911424 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004989368104312939, | |
| "loss": 4.3366, | |
| "theoretical_loss": 5.46394290826635, | |
| "tokens_seen": 39976960 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498926780341023, | |
| "loss": 4.4816, | |
| "theoretical_loss": 5.462542060829209, | |
| "tokens_seen": 40042496 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004989167502507523, | |
| "loss": 4.5005, | |
| "theoretical_loss": 5.461144144994606, | |
| "tokens_seen": 40108032 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004989067201604814, | |
| "loss": 4.5075, | |
| "theoretical_loss": 5.459749149852287, | |
| "tokens_seen": 40173568 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004988966900702107, | |
| "loss": 4.555, | |
| "theoretical_loss": 5.4583570645503166, | |
| "tokens_seen": 40239104 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004988866599799398, | |
| "loss": 4.411, | |
| "theoretical_loss": 5.45696787829467, | |
| "tokens_seen": 40304640 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004988766298896691, | |
| "loss": 4.479, | |
| "theoretical_loss": 5.455581580348831, | |
| "tokens_seen": 40370176 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004988665997993982, | |
| "loss": 4.424, | |
| "theoretical_loss": 5.454198160033391, | |
| "tokens_seen": 40435712 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004988565697091274, | |
| "loss": 4.6216, | |
| "theoretical_loss": 5.452817606725657, | |
| "tokens_seen": 40501248 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004988465396188566, | |
| "loss": 4.5915, | |
| "theoretical_loss": 5.451439909859255, | |
| "tokens_seen": 40566784 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004988365095285858, | |
| "loss": 4.4225, | |
| "theoretical_loss": 5.450065058923747, | |
| "tokens_seen": 40632320 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498826479438315, | |
| "loss": 4.4659, | |
| "theoretical_loss": 5.448693043464235, | |
| "tokens_seen": 40697856 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004988164493480441, | |
| "loss": 4.7231, | |
| "theoretical_loss": 5.447323853080991, | |
| "tokens_seen": 40763392 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004988064192577733, | |
| "loss": 4.561, | |
| "theoretical_loss": 5.445957477429067, | |
| "tokens_seen": 40828928 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004987963891675025, | |
| "loss": 4.4559, | |
| "theoretical_loss": 5.444593906217927, | |
| "tokens_seen": 40894464 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 33820, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.709114074707031, | |
| "objective/train/theoretical_loss": 5.443233129211069, | |
| "objective/train/tokens_used": 61420000, | |
| "theoretical_loss": 5.443233129211069, | |
| "tokens_seen": 40960000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004987863590772317, | |
| "loss": 4.5353, | |
| "theoretical_loss": 5.443233129211069, | |
| "tokens_seen": 40960000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004987763289869609, | |
| "loss": 4.5888, | |
| "theoretical_loss": 5.441875136225658, | |
| "tokens_seen": 41025536 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00049876629889669, | |
| "loss": 4.6609, | |
| "theoretical_loss": 5.440519917132159, | |
| "tokens_seen": 41091072 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004987562688064192, | |
| "loss": 4.6782, | |
| "theoretical_loss": 5.439167461853971, | |
| "tokens_seen": 41156608 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004987462387161484, | |
| "loss": 4.3218, | |
| "theoretical_loss": 5.437817760367073, | |
| "tokens_seen": 41222144 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004987362086258777, | |
| "loss": 4.6102, | |
| "theoretical_loss": 5.436470802699661, | |
| "tokens_seen": 41287680 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004987261785356068, | |
| "loss": 4.4044, | |
| "theoretical_loss": 5.43512657893179, | |
| "tokens_seen": 41353216 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004987161484453361, | |
| "loss": 4.5902, | |
| "theoretical_loss": 5.433785079195035, | |
| "tokens_seen": 41418752 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004987061183550651, | |
| "loss": 4.4395, | |
| "theoretical_loss": 5.432446293672129, | |
| "tokens_seen": 41484288 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004986960882647944, | |
| "loss": 4.6268, | |
| "theoretical_loss": 5.43111021259663, | |
| "tokens_seen": 41549824 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004986860581745236, | |
| "loss": 4.7247, | |
| "theoretical_loss": 5.429776826252565, | |
| "tokens_seen": 41615360 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004986760280842528, | |
| "loss": 4.7844, | |
| "theoretical_loss": 5.428446124974103, | |
| "tokens_seen": 41680896 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498665997993982, | |
| "loss": 4.748, | |
| "theoretical_loss": 5.427118099145207, | |
| "tokens_seen": 41746432 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004986559679037111, | |
| "loss": 4.4521, | |
| "theoretical_loss": 5.425792739199306, | |
| "tokens_seen": 41811968 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004986459378134403, | |
| "loss": 4.5162, | |
| "theoretical_loss": 5.424470035618963, | |
| "tokens_seen": 41877504 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004986359077231695, | |
| "loss": 4.5787, | |
| "theoretical_loss": 5.423149978935547, | |
| "tokens_seen": 41943040 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004986258776328987, | |
| "loss": 4.4447, | |
| "theoretical_loss": 5.421832559728902, | |
| "tokens_seen": 42008576 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004986158475426279, | |
| "loss": 4.5436, | |
| "theoretical_loss": 5.420517768627027, | |
| "tokens_seen": 42074112 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498605817452357, | |
| "loss": 4.5508, | |
| "theoretical_loss": 5.419205596305758, | |
| "tokens_seen": 42139648 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985957873620862, | |
| "loss": 4.3851, | |
| "theoretical_loss": 5.417896033488446, | |
| "tokens_seen": 42205184 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985857572718154, | |
| "loss": 4.4232, | |
| "theoretical_loss": 5.416589070945641, | |
| "tokens_seen": 42270720 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985757271815446, | |
| "loss": 4.4672, | |
| "theoretical_loss": 5.415284699494782, | |
| "tokens_seen": 42336256 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985656970912738, | |
| "loss": 4.3494, | |
| "theoretical_loss": 5.413982909999887, | |
| "tokens_seen": 42401792 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985556670010031, | |
| "loss": 4.246, | |
| "theoretical_loss": 5.412683693371241, | |
| "tokens_seen": 42467328 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985456369107321, | |
| "loss": 4.5265, | |
| "theoretical_loss": 5.411387040565092, | |
| "tokens_seen": 42532864 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 34938, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 3.999885082244873, | |
| "objective/train/theoretical_loss": 5.410092942583357, | |
| "objective/train/tokens_used": 63058400, | |
| "theoretical_loss": 5.410092942583357, | |
| "tokens_seen": 42598400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985356068204614, | |
| "loss": 4.5919, | |
| "theoretical_loss": 5.410092942583357, | |
| "tokens_seen": 42598400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985255767301905, | |
| "loss": 4.603, | |
| "theoretical_loss": 5.408801390473304, | |
| "tokens_seen": 42663936 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985155466399198, | |
| "loss": 4.546, | |
| "theoretical_loss": 5.407512375327272, | |
| "tokens_seen": 42729472 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498505516549649, | |
| "loss": 4.5326, | |
| "theoretical_loss": 5.406225888282363, | |
| "tokens_seen": 42795008 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004984954864593782, | |
| "loss": 4.3889, | |
| "theoretical_loss": 5.404941920520156, | |
| "tokens_seen": 42860544 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004984854563691073, | |
| "loss": 4.1785, | |
| "theoretical_loss": 5.403660463266413, | |
| "tokens_seen": 42926080 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004984754262788365, | |
| "loss": 4.5382, | |
| "theoretical_loss": 5.402381507790792, | |
| "tokens_seen": 42991616 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004984653961885657, | |
| "loss": 4.5693, | |
| "theoretical_loss": 5.401105045406557, | |
| "tokens_seen": 43057152 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004984553660982949, | |
| "loss": 4.4403, | |
| "theoretical_loss": 5.399831067470302, | |
| "tokens_seen": 43122688 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004984453360080241, | |
| "loss": 4.7214, | |
| "theoretical_loss": 5.398559565381662, | |
| "tokens_seen": 43188224 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004984353059177532, | |
| "loss": 4.4442, | |
| "theoretical_loss": 5.397290530583037, | |
| "tokens_seen": 43253760 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004984252758274825, | |
| "loss": 4.309, | |
| "theoretical_loss": 5.396023954559315, | |
| "tokens_seen": 43319296 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004984152457372116, | |
| "loss": 4.3474, | |
| "theoretical_loss": 5.394759828837596, | |
| "tokens_seen": 43384832 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004984052156469409, | |
| "loss": 4.5364, | |
| "theoretical_loss": 5.393498144986921, | |
| "tokens_seen": 43450368 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00049839518555667, | |
| "loss": 4.647, | |
| "theoretical_loss": 5.392238894618, | |
| "tokens_seen": 43515904 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004983851554663993, | |
| "loss": 4.4688, | |
| "theoretical_loss": 5.390982069382944, | |
| "tokens_seen": 43581440 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004983751253761284, | |
| "loss": 4.555, | |
| "theoretical_loss": 5.389727660974998, | |
| "tokens_seen": 43646976 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004983650952858576, | |
| "loss": 4.2673, | |
| "theoretical_loss": 5.38847566112828, | |
| "tokens_seen": 43712512 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004983550651955868, | |
| "loss": 4.223, | |
| "theoretical_loss": 5.387226061617516, | |
| "tokens_seen": 43778048 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498345035105316, | |
| "loss": 4.5944, | |
| "theoretical_loss": 5.38597885425778, | |
| "tokens_seen": 43843584 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004983350050150452, | |
| "loss": 4.2395, | |
| "theoretical_loss": 5.384734030904238, | |
| "tokens_seen": 43909120 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004983249749247743, | |
| "loss": 4.2047, | |
| "theoretical_loss": 5.383491583451892, | |
| "tokens_seen": 43974656 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004983149448345035, | |
| "loss": 4.6784, | |
| "theoretical_loss": 5.382251503835324, | |
| "tokens_seen": 44040192 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004983049147442327, | |
| "loss": 4.5094, | |
| "theoretical_loss": 5.381013784028445, | |
| "tokens_seen": 44105728 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004982948846539619, | |
| "loss": 4.2719, | |
| "theoretical_loss": 5.379778416044247, | |
| "tokens_seen": 44171264 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 35534, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.2067718505859375, | |
| "objective/train/theoretical_loss": 5.378545391934553, | |
| "objective/train/tokens_used": 64696800, | |
| "theoretical_loss": 5.378545391934553, | |
| "tokens_seen": 44236800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004982848545636911, | |
| "loss": 4.3011, | |
| "theoretical_loss": 5.378545391934553, | |
| "tokens_seen": 44236800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004982748244734202, | |
| "loss": 4.3593, | |
| "theoretical_loss": 5.377314703789773, | |
| "tokens_seen": 44302336 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004982647943831494, | |
| "loss": 4.5546, | |
| "theoretical_loss": 5.376086343738656, | |
| "tokens_seen": 44367872 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004982547642928786, | |
| "loss": 4.2113, | |
| "theoretical_loss": 5.3748603039480525, | |
| "tokens_seen": 44433408 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004982447342026079, | |
| "loss": 4.6548, | |
| "theoretical_loss": 5.373636576622671, | |
| "tokens_seen": 44498944 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498234704112337, | |
| "loss": 4.6067, | |
| "theoretical_loss": 5.372415154004842, | |
| "tokens_seen": 44564480 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004982246740220663, | |
| "loss": 4.5434, | |
| "theoretical_loss": 5.371196028374278, | |
| "tokens_seen": 44630016 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004982146439317953, | |
| "loss": 4.219, | |
| "theoretical_loss": 5.369979192047846, | |
| "tokens_seen": 44695552 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004982046138415246, | |
| "loss": 4.4616, | |
| "theoretical_loss": 5.368764637379322, | |
| "tokens_seen": 44761088 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004981945837512538, | |
| "loss": 4.3927, | |
| "theoretical_loss": 5.367552356759175, | |
| "tokens_seen": 44826624 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498184553660983, | |
| "loss": 4.3436, | |
| "theoretical_loss": 5.366342342614322, | |
| "tokens_seen": 44892160 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004981745235707122, | |
| "loss": 4.3641, | |
| "theoretical_loss": 5.365134587407918, | |
| "tokens_seen": 44957696 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004981644934804413, | |
| "loss": 4.3878, | |
| "theoretical_loss": 5.363929083639112, | |
| "tokens_seen": 45023232 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004981544633901705, | |
| "loss": 4.252, | |
| "theoretical_loss": 5.362725823842838, | |
| "tokens_seen": 45088768 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004981444332998997, | |
| "loss": 4.1657, | |
| "theoretical_loss": 5.361524800589581, | |
| "tokens_seen": 45154304 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004981344032096289, | |
| "loss": 4.0417, | |
| "theoretical_loss": 5.360326006485165, | |
| "tokens_seen": 45219840 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004981243731193581, | |
| "loss": 4.6023, | |
| "theoretical_loss": 5.359129434170529, | |
| "tokens_seen": 45285376 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004981143430290873, | |
| "loss": 4.3602, | |
| "theoretical_loss": 5.357935076321514, | |
| "tokens_seen": 45350912 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004981043129388164, | |
| "loss": 4.4446, | |
| "theoretical_loss": 5.35674292564864, | |
| "tokens_seen": 45416448 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004980942828485456, | |
| "loss": 4.4122, | |
| "theoretical_loss": 5.3555529748969, | |
| "tokens_seen": 45481984 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004980842527582748, | |
| "loss": 4.465, | |
| "theoretical_loss": 5.354365216845546, | |
| "tokens_seen": 45547520 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498074222668004, | |
| "loss": 4.1501, | |
| "theoretical_loss": 5.353179644307868, | |
| "tokens_seen": 45613056 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004980641925777333, | |
| "loss": 4.6183, | |
| "theoretical_loss": 5.351996250131005, | |
| "tokens_seen": 45678592 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004980541624874623, | |
| "loss": 4.4826, | |
| "theoretical_loss": 5.350815027195717, | |
| "tokens_seen": 45744128 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004980441323971916, | |
| "loss": 4.0969, | |
| "theoretical_loss": 5.349635968416189, | |
| "tokens_seen": 45809664 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 36791, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 3.983624219894409, | |
| "objective/train/theoretical_loss": 5.34845906673983, | |
| "objective/train/tokens_used": 66335200, | |
| "theoretical_loss": 5.34845906673983, | |
| "tokens_seen": 45875200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004980341023069207, | |
| "loss": 4.1114, | |
| "theoretical_loss": 5.34845906673983, | |
| "tokens_seen": 45875200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00049802407221665, | |
| "loss": 4.3748, | |
| "theoretical_loss": 5.347284315147059, | |
| "tokens_seen": 45940736 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004980140421263792, | |
| "loss": 4.3086, | |
| "theoretical_loss": 5.346111706651114, | |
| "tokens_seen": 46006272 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004980040120361084, | |
| "loss": 4.5521, | |
| "theoretical_loss": 5.344941234297847, | |
| "tokens_seen": 46071808 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979939819458375, | |
| "loss": 4.4492, | |
| "theoretical_loss": 5.343772891165527, | |
| "tokens_seen": 46137344 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979839518555667, | |
| "loss": 4.4528, | |
| "theoretical_loss": 5.34260667036464, | |
| "tokens_seen": 46202880 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979739217652959, | |
| "loss": 4.6275, | |
| "theoretical_loss": 5.3414425650377, | |
| "tokens_seen": 46268416 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979638916750251, | |
| "loss": 4.5955, | |
| "theoretical_loss": 5.3402805683590495, | |
| "tokens_seen": 46333952 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979538615847543, | |
| "loss": 4.5274, | |
| "theoretical_loss": 5.339120673534668, | |
| "tokens_seen": 46399488 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979438314944834, | |
| "loss": 4.1984, | |
| "theoretical_loss": 5.337962873801984, | |
| "tokens_seen": 46465024 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979338014042126, | |
| "loss": 4.4472, | |
| "theoretical_loss": 5.336807162429681, | |
| "tokens_seen": 46530560 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979237713139418, | |
| "loss": 4.2688, | |
| "theoretical_loss": 5.335653532717512, | |
| "tokens_seen": 46596096 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000497913741223671, | |
| "loss": 4.5101, | |
| "theoretical_loss": 5.33450197799611, | |
| "tokens_seen": 46661632 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979037111334002, | |
| "loss": 4.4218, | |
| "theoretical_loss": 5.33335249162681, | |
| "tokens_seen": 46727168 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004978936810431293, | |
| "loss": 4.365, | |
| "theoretical_loss": 5.332205067001454, | |
| "tokens_seen": 46792704 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004978836509528586, | |
| "loss": 4.4012, | |
| "theoretical_loss": 5.331059697542216, | |
| "tokens_seen": 46858240 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004978736208625877, | |
| "loss": 4.178, | |
| "theoretical_loss": 5.329916376701417, | |
| "tokens_seen": 46923776 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000497863590772317, | |
| "loss": 4.404, | |
| "theoretical_loss": 5.328775097961346, | |
| "tokens_seen": 46989312 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004978535606820461, | |
| "loss": 4.3063, | |
| "theoretical_loss": 5.327635854834084, | |
| "tokens_seen": 47054848 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004978435305917754, | |
| "loss": 4.3593, | |
| "theoretical_loss": 5.326498640861321, | |
| "tokens_seen": 47120384 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004978335005015045, | |
| "loss": 4.458, | |
| "theoretical_loss": 5.325363449614185, | |
| "tokens_seen": 47185920 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004978234704112337, | |
| "loss": 4.4505, | |
| "theoretical_loss": 5.324230274693065, | |
| "tokens_seen": 47251456 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004978134403209629, | |
| "loss": 4.2784, | |
| "theoretical_loss": 5.323099109727434, | |
| "tokens_seen": 47316992 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004978034102306921, | |
| "loss": 4.3645, | |
| "theoretical_loss": 5.321969948375687, | |
| "tokens_seen": 47382528 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004977933801404213, | |
| "loss": 4.3349, | |
| "theoretical_loss": 5.320842784324954, | |
| "tokens_seen": 47448064 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 38014, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.285874366760254, | |
| "objective/train/theoretical_loss": 5.319717611290951, | |
| "objective/train/tokens_used": 67973600, | |
| "theoretical_loss": 5.319717611290951, | |
| "tokens_seen": 47513600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004977833500501504, | |
| "loss": 4.2487, | |
| "theoretical_loss": 5.319717611290951, | |
| "tokens_seen": 47513600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004977733199598796, | |
| "loss": 4.227, | |
| "theoretical_loss": 5.318594423017791, | |
| "tokens_seen": 47579136 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004977632898696088, | |
| "loss": 4.5068, | |
| "theoretical_loss": 5.317473213277827, | |
| "tokens_seen": 47644672 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000497753259779338, | |
| "loss": 4.4264, | |
| "theoretical_loss": 5.316353975871486, | |
| "tokens_seen": 47710208 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004977432296890672, | |
| "loss": 4.4848, | |
| "theoretical_loss": 5.315236704627106, | |
| "tokens_seen": 47775744 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004977331995987965, | |
| "loss": 4.4288, | |
| "theoretical_loss": 5.314121393400763, | |
| "tokens_seen": 47841280 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004977231695085255, | |
| "loss": 4.0077, | |
| "theoretical_loss": 5.313008036076115, | |
| "tokens_seen": 47906816 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004977131394182548, | |
| "loss": 4.3322, | |
| "theoretical_loss": 5.311896626564246, | |
| "tokens_seen": 47972352 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000497703109327984, | |
| "loss": 4.1105, | |
| "theoretical_loss": 5.3107871588034925, | |
| "tokens_seen": 48037888 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004976930792377132, | |
| "loss": 4.3437, | |
| "theoretical_loss": 5.309679626759296, | |
| "tokens_seen": 48103424 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004976830491474424, | |
| "loss": 4.292, | |
| "theoretical_loss": 5.308574024424041, | |
| "tokens_seen": 48168960 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004976730190571715, | |
| "loss": 4.3868, | |
| "theoretical_loss": 5.3074703458168955, | |
| "tokens_seen": 48234496 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004976629889669007, | |
| "loss": 4.436, | |
| "theoretical_loss": 5.3063685849836615, | |
| "tokens_seen": 48300032 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004976529588766299, | |
| "loss": 4.368, | |
| "theoretical_loss": 5.305268735996615, | |
| "tokens_seen": 48365568 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004976429287863591, | |
| "loss": 4.2524, | |
| "theoretical_loss": 5.304170792954354, | |
| "tokens_seen": 48431104 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004976328986960883, | |
| "loss": 4.5041, | |
| "theoretical_loss": 5.303074749981648, | |
| "tokens_seen": 48496640 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004976228686058175, | |
| "loss": 4.412, | |
| "theoretical_loss": 5.301980601229285, | |
| "tokens_seen": 48562176 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004976128385155466, | |
| "loss": 4.3484, | |
| "theoretical_loss": 5.300888340873924, | |
| "tokens_seen": 48627712 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004976028084252758, | |
| "loss": 4.4722, | |
| "theoretical_loss": 5.299797963117941, | |
| "tokens_seen": 48693248 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000497592778335005, | |
| "loss": 4.1945, | |
| "theoretical_loss": 5.298709462189283, | |
| "tokens_seen": 48758784 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004975827482447342, | |
| "loss": 4.2003, | |
| "theoretical_loss": 5.297622832341327, | |
| "tokens_seen": 48824320 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004975727181544635, | |
| "loss": 4.0957, | |
| "theoretical_loss": 5.2965380678527225, | |
| "tokens_seen": 48889856 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004975626880641925, | |
| "loss": 4.2418, | |
| "theoretical_loss": 5.295455163027258, | |
| "tokens_seen": 48955392 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004975526579739218, | |
| "loss": 4.3033, | |
| "theoretical_loss": 5.29437411219371, | |
| "tokens_seen": 49020928 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004975426278836509, | |
| "loss": 4.2827, | |
| "theoretical_loss": 5.293294909705702, | |
| "tokens_seen": 49086464 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 38689, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.5455546379089355, | |
| "objective/train/theoretical_loss": 5.292217549941562, | |
| "objective/train/tokens_used": 69612000, | |
| "theoretical_loss": 5.292217549941562, | |
| "tokens_seen": 49152000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004975325977933802, | |
| "loss": 4.2752, | |
| "theoretical_loss": 5.292217549941562, | |
| "tokens_seen": 49152000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004975225677031094, | |
| "loss": 4.1889, | |
| "theoretical_loss": 5.291142027304185, | |
| "tokens_seen": 49217536 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004975125376128386, | |
| "loss": 3.9788, | |
| "theoretical_loss": 5.29006833622089, | |
| "tokens_seen": 49283072 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004975025075225677, | |
| "loss": 4.4666, | |
| "theoretical_loss": 5.28899647114328, | |
| "tokens_seen": 49348608 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004974924774322969, | |
| "loss": 4.3282, | |
| "theoretical_loss": 5.287926426547106, | |
| "tokens_seen": 49414144 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004974824473420261, | |
| "loss": 4.3775, | |
| "theoretical_loss": 5.2868581969321315, | |
| "tokens_seen": 49479680 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004974724172517553, | |
| "loss": 4.3022, | |
| "theoretical_loss": 5.285791776821995, | |
| "tokens_seen": 49545216 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004974623871614845, | |
| "loss": 4.559, | |
| "theoretical_loss": 5.2847271607640725, | |
| "tokens_seen": 49610752 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004974523570712136, | |
| "loss": 4.5333, | |
| "theoretical_loss": 5.283664343329346, | |
| "tokens_seen": 49676288 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004974423269809428, | |
| "loss": 4.2267, | |
| "theoretical_loss": 5.282603319112273, | |
| "tokens_seen": 49741824 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000497432296890672, | |
| "loss": 4.3567, | |
| "theoretical_loss": 5.281544082730649, | |
| "tokens_seen": 49807360 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004974222668004012, | |
| "loss": 4.2938, | |
| "theoretical_loss": 5.280486628825479, | |
| "tokens_seen": 49872896 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004974122367101304, | |
| "loss": 4.3754, | |
| "theoretical_loss": 5.279430952060843, | |
| "tokens_seen": 49938432 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004974022066198595, | |
| "loss": 4.3361, | |
| "theoretical_loss": 5.278377047123777, | |
| "tokens_seen": 50003968 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973921765295888, | |
| "loss": 4.2186, | |
| "theoretical_loss": 5.277324908724132, | |
| "tokens_seen": 50069504 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973821464393179, | |
| "loss": 4.2978, | |
| "theoretical_loss": 5.276274531594453, | |
| "tokens_seen": 50135040 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973721163490472, | |
| "loss": 4.3052, | |
| "theoretical_loss": 5.27522591048985, | |
| "tokens_seen": 50200576 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973620862587763, | |
| "loss": 4.352, | |
| "theoretical_loss": 5.274179040187876, | |
| "tokens_seen": 50266112 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973520561685056, | |
| "loss": 4.0427, | |
| "theoretical_loss": 5.273133915488394, | |
| "tokens_seen": 50331648 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973420260782347, | |
| "loss": 4.3078, | |
| "theoretical_loss": 5.272090531213462, | |
| "tokens_seen": 50397184 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973319959879639, | |
| "loss": 4.0645, | |
| "theoretical_loss": 5.271048882207202, | |
| "tokens_seen": 50462720 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973219658976931, | |
| "loss": 4.6409, | |
| "theoretical_loss": 5.27000896333568, | |
| "tokens_seen": 50528256 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973119358074223, | |
| "loss": 4.0496, | |
| "theoretical_loss": 5.268970769486789, | |
| "tokens_seen": 50593792 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973019057171515, | |
| "loss": 4.4574, | |
| "theoretical_loss": 5.267934295570118, | |
| "tokens_seen": 50659328 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004972918756268806, | |
| "loss": 4.2343, | |
| "theoretical_loss": 5.266899536516842, | |
| "tokens_seen": 50724864 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "objective/train/docs_used": 39885, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.198443412780762, | |
| "objective/train/theoretical_loss": 5.265866487279597, | |
| "objective/train/tokens_used": 71250400, | |
| "theoretical_loss": 5.265866487279597, | |
| "tokens_seen": 50790400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004972818455366098, | |
| "loss": 4.2608, | |
| "theoretical_loss": 5.265866487279597, | |
| "tokens_seen": 50790400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000497271815446339, | |
| "loss": 4.2763, | |
| "theoretical_loss": 5.264835142832364, | |
| "tokens_seen": 50855936 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004972617853560682, | |
| "loss": 4.1906, | |
| "theoretical_loss": 5.263805498170351, | |
| "tokens_seen": 50921472 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004972517552657974, | |
| "loss": 4.1761, | |
| "theoretical_loss": 5.26277754830987, | |
| "tokens_seen": 50987008 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004972417251755266, | |
| "loss": 4.2526, | |
| "theoretical_loss": 5.261751288288236, | |
| "tokens_seen": 51052544 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004972316950852557, | |
| "loss": 4.275, | |
| "theoretical_loss": 5.260726713163637, | |
| "tokens_seen": 51118080 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004972216649949849, | |
| "loss": 4.3822, | |
| "theoretical_loss": 5.259703818015023, | |
| "tokens_seen": 51183616 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004972116349047142, | |
| "loss": 4.235, | |
| "theoretical_loss": 5.258682597941998, | |
| "tokens_seen": 51249152 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004972016048144433, | |
| "loss": 4.1933, | |
| "theoretical_loss": 5.257663048064704, | |
| "tokens_seen": 51314688 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004971915747241726, | |
| "loss": 4.0819, | |
| "theoretical_loss": 5.256645163523704, | |
| "tokens_seen": 51380224 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004971815446339017, | |
| "loss": 4.2302, | |
| "theoretical_loss": 5.255628939479877, | |
| "tokens_seen": 51445760 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004971715145436309, | |
| "loss": 4.1741, | |
| "theoretical_loss": 5.254614371114306, | |
| "tokens_seen": 51511296 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004971614844533601, | |
| "loss": 4.0105, | |
| "theoretical_loss": 5.253601453628165, | |
| "tokens_seen": 51576832 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004971514543630893, | |
| "loss": 4.1921, | |
| "theoretical_loss": 5.2525901822426135, | |
| "tokens_seen": 51642368 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004971414242728185, | |
| "loss": 4.1033, | |
| "theoretical_loss": 5.251580552198687, | |
| "tokens_seen": 51707904 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004971313941825477, | |
| "loss": 4.5223, | |
| "theoretical_loss": 5.250572558757186, | |
| "tokens_seen": 51773440 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004971213640922768, | |
| "loss": 4.2685, | |
| "theoretical_loss": 5.2495661971985745, | |
| "tokens_seen": 51838976 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000497111334002006, | |
| "loss": 4.4627, | |
| "theoretical_loss": 5.248561462822871, | |
| "tokens_seen": 51904512 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004971013039117352, | |
| "loss": 4.2511, | |
| "theoretical_loss": 5.247558350949539, | |
| "tokens_seen": 51970048 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004970912738214644, | |
| "loss": 4.0554, | |
| "theoretical_loss": 5.246556856917392, | |
| "tokens_seen": 52035584 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004970812437311936, | |
| "loss": 4.1646, | |
| "theoretical_loss": 5.24555697608448, | |
| "tokens_seen": 52101120 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004970712136409227, | |
| "loss": 4.2295, | |
| "theoretical_loss": 5.244558703827986, | |
| "tokens_seen": 52166656 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004970611835506519, | |
| "loss": 4.1183, | |
| "theoretical_loss": 5.243562035544132, | |
| "tokens_seen": 52232192 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004970511534603811, | |
| "loss": 4.4799, | |
| "theoretical_loss": 5.24256696664807, | |
| "tokens_seen": 52297728 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004970411233701103, | |
| "loss": 4.1907, | |
| "theoretical_loss": 5.24157349257378, | |
| "tokens_seen": 52363264 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "objective/train/docs_used": 40418, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 3.979517936706543, | |
| "objective/train/theoretical_loss": 5.240581608773973, | |
| "objective/train/tokens_used": 72888800, | |
| "theoretical_loss": 5.240581608773973, | |
| "tokens_seen": 52428800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004970310932798396, | |
| "loss": 4.0593, | |
| "theoretical_loss": 5.240581608773973, | |
| "tokens_seen": 52428800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004970210631895686, | |
| "loss": 4.2029, | |
| "theoretical_loss": 5.239591310719987, | |
| "tokens_seen": 52494336 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004970110330992979, | |
| "loss": 4.0965, | |
| "theoretical_loss": 5.238602593901694, | |
| "tokens_seen": 52559872 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004970010030090271, | |
| "loss": 4.0066, | |
| "theoretical_loss": 5.23761545382739, | |
| "tokens_seen": 52625408 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004969909729187563, | |
| "loss": 4.1287, | |
| "theoretical_loss": 5.236629886023708, | |
| "tokens_seen": 52690944 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004969809428284855, | |
| "loss": 4.1441, | |
| "theoretical_loss": 5.235645886035517, | |
| "tokens_seen": 52756480 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004969709127382147, | |
| "loss": 4.2684, | |
| "theoretical_loss": 5.2346634494258195, | |
| "tokens_seen": 52822016 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004969608826479438, | |
| "loss": 4.1129, | |
| "theoretical_loss": 5.233682571775664, | |
| "tokens_seen": 52887552 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000496950852557673, | |
| "loss": 4.44, | |
| "theoretical_loss": 5.232703248684045, | |
| "tokens_seen": 52953088 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004969408224674022, | |
| "loss": 4.3805, | |
| "theoretical_loss": 5.231725475767806, | |
| "tokens_seen": 53018624 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004969307923771314, | |
| "loss": 4.012, | |
| "theoretical_loss": 5.2307492486615494, | |
| "tokens_seen": 53084160 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004969207622868606, | |
| "loss": 4.3914, | |
| "theoretical_loss": 5.229774563017541, | |
| "tokens_seen": 53149696 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004969107321965897, | |
| "loss": 4.2825, | |
| "theoretical_loss": 5.2288014145056145, | |
| "tokens_seen": 53215232 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000496900702106319, | |
| "loss": 3.9345, | |
| "theoretical_loss": 5.227829798813083, | |
| "tokens_seen": 53280768 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004968906720160481, | |
| "loss": 4.1139, | |
| "theoretical_loss": 5.2268597116446465, | |
| "tokens_seen": 53346304 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004968806419257774, | |
| "loss": 4.2213, | |
| "theoretical_loss": 5.225891148722296, | |
| "tokens_seen": 53411840 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004968706118355065, | |
| "loss": 4.1851, | |
| "theoretical_loss": 5.224924105785227, | |
| "tokens_seen": 53477376 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004968605817452358, | |
| "loss": 4.1925, | |
| "theoretical_loss": 5.22395857858975, | |
| "tokens_seen": 53542912 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004968505516549649, | |
| "loss": 3.8839, | |
| "theoretical_loss": 5.222994562909199, | |
| "tokens_seen": 53608448 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004968405215646941, | |
| "loss": 4.1607, | |
| "theoretical_loss": 5.222032054533841, | |
| "tokens_seen": 53673984 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004968304914744233, | |
| "loss": 4.2475, | |
| "theoretical_loss": 5.2210710492707895, | |
| "tokens_seen": 53739520 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004968204613841525, | |
| "loss": 3.881, | |
| "theoretical_loss": 5.220111542943918, | |
| "tokens_seen": 53805056 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004968104312938817, | |
| "loss": 4.0171, | |
| "theoretical_loss": 5.21915353139377, | |
| "tokens_seen": 53870592 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004968004012036108, | |
| "loss": 4.0926, | |
| "theoretical_loss": 5.218197010477477, | |
| "tokens_seen": 53936128 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.00049679037111334, | |
| "loss": 4.3292, | |
| "theoretical_loss": 5.21724197606866, | |
| "tokens_seen": 54001664 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "objective/train/docs_used": 41741, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.073785305023193, | |
| "objective/train/theoretical_loss": 5.216288424057362, | |
| "objective/train/tokens_used": 74527200, | |
| "theoretical_loss": 5.216288424057362, | |
| "tokens_seen": 54067200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967803410230692, | |
| "loss": 4.2491, | |
| "theoretical_loss": 5.216288424057362, | |
| "tokens_seen": 54067200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967703109327984, | |
| "loss": 4.2446, | |
| "theoretical_loss": 5.21533635034995, | |
| "tokens_seen": 54132736 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967602808425276, | |
| "loss": 4.2944, | |
| "theoretical_loss": 5.214385750869031, | |
| "tokens_seen": 54198272 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967502507522568, | |
| "loss": 4.1296, | |
| "theoretical_loss": 5.213436621553377, | |
| "tokens_seen": 54263808 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967402206619859, | |
| "loss": 4.17, | |
| "theoretical_loss": 5.212488958357831, | |
| "tokens_seen": 54329344 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967301905717151, | |
| "loss": 4.0717, | |
| "theoretical_loss": 5.211542757253229, | |
| "tokens_seen": 54394880 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967201604814444, | |
| "loss": 4.4436, | |
| "theoretical_loss": 5.210598014226319, | |
| "tokens_seen": 54460416 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967101303911735, | |
| "loss": 4.1394, | |
| "theoretical_loss": 5.209654725279678, | |
| "tokens_seen": 54525952 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967001003009028, | |
| "loss": 4.4479, | |
| "theoretical_loss": 5.208712886431626, | |
| "tokens_seen": 54591488 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000496690070210632, | |
| "loss": 3.9774, | |
| "theoretical_loss": 5.207772493716154, | |
| "tokens_seen": 54657024 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004966800401203611, | |
| "loss": 4.0348, | |
| "theoretical_loss": 5.206833543182833, | |
| "tokens_seen": 54722560 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004966700100300903, | |
| "loss": 4.1275, | |
| "theoretical_loss": 5.205896030896748, | |
| "tokens_seen": 54788096 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004966599799398195, | |
| "loss": 4.034, | |
| "theoretical_loss": 5.2049599529384025, | |
| "tokens_seen": 54853632 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004966499498495487, | |
| "loss": 4.2901, | |
| "theoretical_loss": 5.204025305403653, | |
| "tokens_seen": 54919168 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004966399197592779, | |
| "loss": 4.0859, | |
| "theoretical_loss": 5.2030920844036235, | |
| "tokens_seen": 54984704 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000496629889669007, | |
| "loss": 4.2186, | |
| "theoretical_loss": 5.202160286064628, | |
| "tokens_seen": 55050240 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004966198595787362, | |
| "loss": 4.1753, | |
| "theoretical_loss": 5.2012299065281, | |
| "tokens_seen": 55115776 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004966098294884654, | |
| "loss": 4.2037, | |
| "theoretical_loss": 5.2003009419505055, | |
| "tokens_seen": 55181312 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004965997993981946, | |
| "loss": 4.3058, | |
| "theoretical_loss": 5.199373388503272, | |
| "tokens_seen": 55246848 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004965897693079238, | |
| "loss": 4.3641, | |
| "theoretical_loss": 5.198447242372716, | |
| "tokens_seen": 55312384 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004965797392176529, | |
| "loss": 4.1662, | |
| "theoretical_loss": 5.197522499759961, | |
| "tokens_seen": 55377920 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004965697091273821, | |
| "loss": 4.1908, | |
| "theoretical_loss": 5.196599156880863, | |
| "tokens_seen": 55443456 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004965596790371113, | |
| "loss": 3.9863, | |
| "theoretical_loss": 5.195677209965942, | |
| "tokens_seen": 55508992 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004965496489468405, | |
| "loss": 4.5222, | |
| "theoretical_loss": 5.194756655260305, | |
| "tokens_seen": 55574528 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004965396188565698, | |
| "loss": 3.9698, | |
| "theoretical_loss": 5.193837489023565, | |
| "tokens_seen": 55640064 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "objective/train/docs_used": 42409, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.305491924285889, | |
| "objective/train/theoretical_loss": 5.192919707529784, | |
| "objective/train/tokens_used": 76165600, | |
| "theoretical_loss": 5.192919707529784, | |
| "tokens_seen": 55705600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004965295887662988, | |
| "loss": 4.121, | |
| "theoretical_loss": 5.192919707529784, | |
| "tokens_seen": 55705600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004965195586760281, | |
| "loss": 4.108, | |
| "theoretical_loss": 5.1920033070673846, | |
| "tokens_seen": 55771136 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004965095285857573, | |
| "loss": 3.9647, | |
| "theoretical_loss": 5.191088283939086, | |
| "tokens_seen": 55836672 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004964994984954865, | |
| "loss": 3.9453, | |
| "theoretical_loss": 5.190174634461828, | |
| "tokens_seen": 55902208 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004964894684052157, | |
| "loss": 3.9193, | |
| "theoretical_loss": 5.189262354966708, | |
| "tokens_seen": 55967744 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004964794383149449, | |
| "loss": 4.0925, | |
| "theoretical_loss": 5.188351441798897, | |
| "tokens_seen": 56033280 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000496469408224674, | |
| "loss": 4.1099, | |
| "theoretical_loss": 5.187441891317581, | |
| "tokens_seen": 56098816 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004964593781344032, | |
| "loss": 4.0055, | |
| "theoretical_loss": 5.186533699895888, | |
| "tokens_seen": 56164352 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004964493480441324, | |
| "loss": 4.1547, | |
| "theoretical_loss": 5.18562686392081, | |
| "tokens_seen": 56229888 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004964393179538616, | |
| "loss": 4.1385, | |
| "theoretical_loss": 5.184721379793145, | |
| "tokens_seen": 56295424 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004964292878635908, | |
| "loss": 4.1137, | |
| "theoretical_loss": 5.183817243927425, | |
| "tokens_seen": 56360960 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.00049641925777332, | |
| "loss": 3.988, | |
| "theoretical_loss": 5.182914452751846, | |
| "tokens_seen": 56426496 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004964092276830491, | |
| "loss": 3.9087, | |
| "theoretical_loss": 5.1820130027081985, | |
| "tokens_seen": 56492032 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004963991975927783, | |
| "loss": 4.1078, | |
| "theoretical_loss": 5.181112890251804, | |
| "tokens_seen": 56557568 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004963891675025075, | |
| "loss": 3.9657, | |
| "theoretical_loss": 5.180214111851447, | |
| "tokens_seen": 56623104 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004963791374122367, | |
| "loss": 3.8969, | |
| "theoretical_loss": 5.179316663989305, | |
| "tokens_seen": 56688640 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004963691073219659, | |
| "loss": 4.0938, | |
| "theoretical_loss": 5.178420543160889, | |
| "tokens_seen": 56754176 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004963590772316951, | |
| "loss": 4.0717, | |
| "theoretical_loss": 5.177525745874968, | |
| "tokens_seen": 56819712 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004963490471414242, | |
| "loss": 4.0293, | |
| "theoretical_loss": 5.176632268653516, | |
| "tokens_seen": 56885248 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004963390170511535, | |
| "loss": 4.0914, | |
| "theoretical_loss": 5.175740108031633, | |
| "tokens_seen": 56950784 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004963289869608827, | |
| "loss": 4.3386, | |
| "theoretical_loss": 5.174849260557494, | |
| "tokens_seen": 57016320 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004963189568706119, | |
| "loss": 4.1438, | |
| "theoretical_loss": 5.173959722792272, | |
| "tokens_seen": 57081856 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000496308926780341, | |
| "loss": 4.1502, | |
| "theoretical_loss": 5.173071491310086, | |
| "tokens_seen": 57147392 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004962988966900702, | |
| "loss": 3.9246, | |
| "theoretical_loss": 5.172184562697928, | |
| "tokens_seen": 57212928 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004962888665997994, | |
| "loss": 4.0249, | |
| "theoretical_loss": 5.171298933555606, | |
| "tokens_seen": 57278464 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "objective/train/docs_used": 43695, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 3.649152994155884, | |
| "objective/train/theoretical_loss": 5.170414600495677, | |
| "objective/train/tokens_used": 77804000, | |
| "theoretical_loss": 5.170414600495677, | |
| "tokens_seen": 57344000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004962788365095286, | |
| "loss": 4.0093, | |
| "theoretical_loss": 5.170414600495677, | |
| "tokens_seen": 57344000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004962688064192578, | |
| "loss": 4.1786, | |
| "theoretical_loss": 5.16953156014339, | |
| "tokens_seen": 57409536 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000496258776328987, | |
| "loss": 4.0607, | |
| "theoretical_loss": 5.1686498091366175, | |
| "tokens_seen": 57475072 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004962487462387161, | |
| "loss": 3.6266, | |
| "theoretical_loss": 5.1677693441258, | |
| "tokens_seen": 57540608 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004962387161484453, | |
| "loss": 3.8942, | |
| "theoretical_loss": 5.16689016177388, | |
| "tokens_seen": 57606144 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004962286860581746, | |
| "loss": 3.9167, | |
| "theoretical_loss": 5.166012258756243, | |
| "tokens_seen": 57671680 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004962186559679037, | |
| "loss": 4.2399, | |
| "theoretical_loss": 5.16513563176066, | |
| "tokens_seen": 57737216 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000496208625877633, | |
| "loss": 3.7377, | |
| "theoretical_loss": 5.164260277487221, | |
| "tokens_seen": 57802752 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961985957873621, | |
| "loss": 4.0589, | |
| "theoretical_loss": 5.163386192648282, | |
| "tokens_seen": 57868288 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961885656970913, | |
| "loss": 4.2436, | |
| "theoretical_loss": 5.162513373968403, | |
| "tokens_seen": 57933824 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961785356068205, | |
| "loss": 4.2688, | |
| "theoretical_loss": 5.1616418181842825, | |
| "tokens_seen": 57999360 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961685055165497, | |
| "loss": 3.9842, | |
| "theoretical_loss": 5.160771522044712, | |
| "tokens_seen": 58064896 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961584754262789, | |
| "loss": 4.202, | |
| "theoretical_loss": 5.159902482310506, | |
| "tokens_seen": 58130432 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961484453360081, | |
| "loss": 4.0252, | |
| "theoretical_loss": 5.15903469575445, | |
| "tokens_seen": 58195968 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961384152457372, | |
| "loss": 4.0653, | |
| "theoretical_loss": 5.1581681591612405, | |
| "tokens_seen": 58261504 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961283851554664, | |
| "loss": 4.1853, | |
| "theoretical_loss": 5.15730286932743, | |
| "tokens_seen": 58327040 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961183550651956, | |
| "loss": 4.0287, | |
| "theoretical_loss": 5.156438823061366, | |
| "tokens_seen": 58392576 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961083249749248, | |
| "loss": 3.8706, | |
| "theoretical_loss": 5.15557601718314, | |
| "tokens_seen": 58458112 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000496098294884654, | |
| "loss": 3.4818, | |
| "theoretical_loss": 5.154714448524525, | |
| "tokens_seen": 58523648 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004960882647943831, | |
| "loss": 4.1847, | |
| "theoretical_loss": 5.153854113928924, | |
| "tokens_seen": 58589184 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004960782347041123, | |
| "loss": 4.0123, | |
| "theoretical_loss": 5.152995010251313, | |
| "tokens_seen": 58654720 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004960682046138415, | |
| "loss": 3.8546, | |
| "theoretical_loss": 5.1521371343581865, | |
| "tokens_seen": 58720256 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004960581745235707, | |
| "loss": 3.9651, | |
| "theoretical_loss": 5.1512804831275, | |
| "tokens_seen": 58785792 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004960481444333, | |
| "loss": 3.9367, | |
| "theoretical_loss": 5.1504250534486165, | |
| "tokens_seen": 58851328 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000496038114343029, | |
| "loss": 4.3063, | |
| "theoretical_loss": 5.149570842222257, | |
| "tokens_seen": 58916864 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "objective/train/docs_used": 44395, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.063347339630127, | |
| "objective/train/theoretical_loss": 5.1487178463604355, | |
| "objective/train/tokens_used": 79442400, | |
| "theoretical_loss": 5.1487178463604355, | |
| "tokens_seen": 58982400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004960280842527583, | |
| "loss": 3.7506, | |
| "theoretical_loss": 5.1487178463604355, | |
| "tokens_seen": 58982400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004960180541624875, | |
| "loss": 3.9872, | |
| "theoretical_loss": 5.147866062786418, | |
| "tokens_seen": 59047936 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004960080240722167, | |
| "loss": 3.9911, | |
| "theoretical_loss": 5.147015488434661, | |
| "tokens_seen": 59113472 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004959979939819459, | |
| "loss": 3.8278, | |
| "theoretical_loss": 5.14616612025076, | |
| "tokens_seen": 59179008 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004959879638916751, | |
| "loss": 3.9528, | |
| "theoretical_loss": 5.145317955191397, | |
| "tokens_seen": 59244544 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004959779338014042, | |
| "loss": 3.7919, | |
| "theoretical_loss": 5.144470990224288, | |
| "tokens_seen": 59310080 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004959679037111334, | |
| "loss": 3.9634, | |
| "theoretical_loss": 5.143625222328134, | |
| "tokens_seen": 59375616 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004959578736208626, | |
| "loss": 4.096, | |
| "theoretical_loss": 5.142780648492563, | |
| "tokens_seen": 59441152 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004959478435305918, | |
| "loss": 3.9749, | |
| "theoretical_loss": 5.141937265718084, | |
| "tokens_seen": 59506688 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000495937813440321, | |
| "loss": 4.0214, | |
| "theoretical_loss": 5.141095071016032, | |
| "tokens_seen": 59572224 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004959277833500501, | |
| "loss": 3.9143, | |
| "theoretical_loss": 5.14025406140852, | |
| "tokens_seen": 59637760 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004959177532597793, | |
| "loss": 4.0642, | |
| "theoretical_loss": 5.139414233928385, | |
| "tokens_seen": 59703296 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004959077231695085, | |
| "loss": 3.9695, | |
| "theoretical_loss": 5.138575585619142, | |
| "tokens_seen": 59768832 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004958976930792377, | |
| "loss": 4.1946, | |
| "theoretical_loss": 5.13773811353493, | |
| "tokens_seen": 59834368 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004958876629889669, | |
| "loss": 4.0784, | |
| "theoretical_loss": 5.1369018147404635, | |
| "tokens_seen": 59899904 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000495877632898696, | |
| "loss": 4.0826, | |
| "theoretical_loss": 5.136066686310986, | |
| "tokens_seen": 59965440 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004958676028084253, | |
| "loss": 4.0869, | |
| "theoretical_loss": 5.135232725332212, | |
| "tokens_seen": 60030976 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004958575727181544, | |
| "loss": 3.9758, | |
| "theoretical_loss": 5.134399928900292, | |
| "tokens_seen": 60096512 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004958475426278837, | |
| "loss": 4.0519, | |
| "theoretical_loss": 5.13356829412175, | |
| "tokens_seen": 60162048 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004958375125376129, | |
| "loss": 4.1545, | |
| "theoretical_loss": 5.132737818113444, | |
| "tokens_seen": 60227584 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004958274824473421, | |
| "loss": 4.2355, | |
| "theoretical_loss": 5.131908498002513, | |
| "tokens_seen": 60293120 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004958174523570712, | |
| "loss": 3.8189, | |
| "theoretical_loss": 5.131080330926334, | |
| "tokens_seen": 60358656 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004958074222668004, | |
| "loss": 4.1869, | |
| "theoretical_loss": 5.130253314032469, | |
| "tokens_seen": 60424192 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004957973921765296, | |
| "loss": 3.728, | |
| "theoretical_loss": 5.1294274444786225, | |
| "tokens_seen": 60489728 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004957873620862588, | |
| "loss": 4.1881, | |
| "theoretical_loss": 5.128602719432592, | |
| "tokens_seen": 60555264 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "objective/train/docs_used": 45571, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.731624603271484, | |
| "objective/train/theoretical_loss": 5.12777913607222, | |
| "objective/train/tokens_used": 81080800, | |
| "theoretical_loss": 5.12777913607222, | |
| "tokens_seen": 60620800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000495777331995988, | |
| "loss": 4.3579, | |
| "theoretical_loss": 5.12777913607222, | |
| "tokens_seen": 60620800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004957673019057172, | |
| "loss": 3.8709, | |
| "theoretical_loss": 5.126956691585351, | |
| "tokens_seen": 60686336 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004957572718154463, | |
| "loss": 3.9854, | |
| "theoretical_loss": 5.1261353831697845, | |
| "tokens_seen": 60751872 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004957472417251755, | |
| "loss": 4.2274, | |
| "theoretical_loss": 5.125315208033226, | |
| "tokens_seen": 60817408 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004957372116349047, | |
| "loss": 3.8993, | |
| "theoretical_loss": 5.124496163393244, | |
| "tokens_seen": 60882944 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004957271815446339, | |
| "loss": 3.7806, | |
| "theoretical_loss": 5.123678246477226, | |
| "tokens_seen": 60948480 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004957171514543631, | |
| "loss": 3.977, | |
| "theoretical_loss": 5.122861454522329, | |
| "tokens_seen": 61014016 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004957071213640923, | |
| "loss": 3.9739, | |
| "theoretical_loss": 5.122045784775439, | |
| "tokens_seen": 61079552 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956970912738214, | |
| "loss": 3.8256, | |
| "theoretical_loss": 5.1212312344931235, | |
| "tokens_seen": 61145088 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956870611835507, | |
| "loss": 4.0096, | |
| "theoretical_loss": 5.120417800941587, | |
| "tokens_seen": 61210624 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956770310932798, | |
| "loss": 4.0399, | |
| "theoretical_loss": 5.119605481396629, | |
| "tokens_seen": 61276160 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956670010030091, | |
| "loss": 3.9718, | |
| "theoretical_loss": 5.1187942731436, | |
| "tokens_seen": 61341696 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956569709127383, | |
| "loss": 3.9982, | |
| "theoretical_loss": 5.117984173477356, | |
| "tokens_seen": 61407232 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956469408224674, | |
| "loss": 4.0777, | |
| "theoretical_loss": 5.117175179702215, | |
| "tokens_seen": 61472768 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956369107321966, | |
| "loss": 3.9865, | |
| "theoretical_loss": 5.116367289131917, | |
| "tokens_seen": 61538304 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956268806419258, | |
| "loss": 4.1915, | |
| "theoretical_loss": 5.115560499089575, | |
| "tokens_seen": 61603840 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000495616850551655, | |
| "loss": 3.8813, | |
| "theoretical_loss": 5.1147548069076425, | |
| "tokens_seen": 61669376 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956068204613842, | |
| "loss": 3.97, | |
| "theoretical_loss": 5.113950209927859, | |
| "tokens_seen": 61734912 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004955967903711133, | |
| "loss": 3.5137, | |
| "theoretical_loss": 5.113146705501217, | |
| "tokens_seen": 61800448 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004955867602808425, | |
| "loss": 3.7782, | |
| "theoretical_loss": 5.112344290987915, | |
| "tokens_seen": 61865984 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004955767301905717, | |
| "loss": 4.1055, | |
| "theoretical_loss": 5.1115429637573175, | |
| "tokens_seen": 61931520 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004955667001003009, | |
| "loss": 3.9588, | |
| "theoretical_loss": 5.110742721187915, | |
| "tokens_seen": 61997056 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004955566700100301, | |
| "loss": 3.8558, | |
| "theoretical_loss": 5.109943560667279, | |
| "tokens_seen": 62062592 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004955466399197592, | |
| "loss": 3.8969, | |
| "theoretical_loss": 5.109145479592028, | |
| "tokens_seen": 62128128 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004955366098294884, | |
| "loss": 4.1078, | |
| "theoretical_loss": 5.108348475367774, | |
| "tokens_seen": 62193664 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "objective/train/docs_used": 46598, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.215669631958008, | |
| "objective/train/theoretical_loss": 5.107552545409097, | |
| "objective/train/tokens_used": 82719200, | |
| "theoretical_loss": 5.107552545409097, | |
| "tokens_seen": 62259200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004955265797392177, | |
| "loss": 4.056, | |
| "theoretical_loss": 5.107552545409097, | |
| "tokens_seen": 62259200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004955165496489468, | |
| "loss": 3.9227, | |
| "theoretical_loss": 5.106757687139494, | |
| "tokens_seen": 62324736 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004955065195586761, | |
| "loss": 4.2246, | |
| "theoretical_loss": 5.1059638979913435, | |
| "tokens_seen": 62390272 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004954964894684052, | |
| "loss": 4.0438, | |
| "theoretical_loss": 5.105171175405864, | |
| "tokens_seen": 62455808 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004954864593781344, | |
| "loss": 4.0275, | |
| "theoretical_loss": 5.104379516833075, | |
| "tokens_seen": 62521344 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004954764292878636, | |
| "loss": 4.005, | |
| "theoretical_loss": 5.103588919731757, | |
| "tokens_seen": 62586880 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004954663991975928, | |
| "loss": 3.937, | |
| "theoretical_loss": 5.1027993815694135, | |
| "tokens_seen": 62652416 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000495456369107322, | |
| "loss": 4.1536, | |
| "theoretical_loss": 5.10201089982223, | |
| "tokens_seen": 62717952 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004954463390170512, | |
| "loss": 3.971, | |
| "theoretical_loss": 5.101223471975038, | |
| "tokens_seen": 62783488 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004954363089267803, | |
| "loss": 3.9062, | |
| "theoretical_loss": 5.100437095521272, | |
| "tokens_seen": 62849024 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004954262788365095, | |
| "loss": 3.8866, | |
| "theoretical_loss": 5.099651767962936, | |
| "tokens_seen": 62914560 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004954162487462387, | |
| "loss": 3.8535, | |
| "theoretical_loss": 5.098867486810562, | |
| "tokens_seen": 62980096 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004954062186559679, | |
| "loss": 3.7859, | |
| "theoretical_loss": 5.098084249583175, | |
| "tokens_seen": 63045632 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004953961885656971, | |
| "loss": 3.9093, | |
| "theoretical_loss": 5.097302053808251, | |
| "tokens_seen": 63111168 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004953861584754263, | |
| "loss": 3.5672, | |
| "theoretical_loss": 5.0965208970216835, | |
| "tokens_seen": 63176704 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004953761283851555, | |
| "loss": 3.8846, | |
| "theoretical_loss": 5.095740776767743, | |
| "tokens_seen": 63242240 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004953660982948846, | |
| "loss": 3.9469, | |
| "theoretical_loss": 5.0949616905990425, | |
| "tokens_seen": 63307776 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004953560682046139, | |
| "loss": 4.2765, | |
| "theoretical_loss": 5.094183636076501, | |
| "tokens_seen": 63373312 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004953460381143431, | |
| "loss": 4.015, | |
| "theoretical_loss": 5.093406610769302, | |
| "tokens_seen": 63438848 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004953360080240723, | |
| "loss": 4.2246, | |
| "theoretical_loss": 5.092630612254862, | |
| "tokens_seen": 63504384 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004953259779338014, | |
| "loss": 3.9173, | |
| "theoretical_loss": 5.091855638118792, | |
| "tokens_seen": 63569920 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004953159478435306, | |
| "loss": 4.0249, | |
| "theoretical_loss": 5.091081685954863, | |
| "tokens_seen": 63635456 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004953059177532598, | |
| "loss": 3.7978, | |
| "theoretical_loss": 5.090308753364967, | |
| "tokens_seen": 63700992 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000495295887662989, | |
| "loss": 3.8605, | |
| "theoretical_loss": 5.089536837959084, | |
| "tokens_seen": 63766528 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004952858575727182, | |
| "loss": 3.9657, | |
| "theoretical_loss": 5.0887659373552445, | |
| "tokens_seen": 63832064 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "objective/train/docs_used": 47189, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 3.3733479976654053, | |
| "objective/train/theoretical_loss": 5.087996049179497, | |
| "objective/train/tokens_used": 84357600, | |
| "theoretical_loss": 5.087996049179497, | |
| "tokens_seen": 63897600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004952758274824474, | |
| "loss": 3.6824, | |
| "theoretical_loss": 5.087996049179497, | |
| "tokens_seen": 63897600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004952657973921765, | |
| "loss": 3.7677, | |
| "theoretical_loss": 5.087227171065869, | |
| "tokens_seen": 63963136 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004952557673019057, | |
| "loss": 3.9197, | |
| "theoretical_loss": 5.086459300656337, | |
| "tokens_seen": 64028672 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004952457372116349, | |
| "loss": 3.927, | |
| "theoretical_loss": 5.085692435600783, | |
| "tokens_seen": 64094208 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004952357071213641, | |
| "loss": 3.8821, | |
| "theoretical_loss": 5.084926573556972, | |
| "tokens_seen": 64159744 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004952256770310933, | |
| "loss": 3.7931, | |
| "theoretical_loss": 5.084161712190507, | |
| "tokens_seen": 64225280 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004952156469408225, | |
| "loss": 3.7647, | |
| "theoretical_loss": 5.083397849174801, | |
| "tokens_seen": 64290816 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004952056168505516, | |
| "loss": 3.9431, | |
| "theoretical_loss": 5.082634982191042, | |
| "tokens_seen": 64356352 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004951955867602809, | |
| "loss": 3.9173, | |
| "theoretical_loss": 5.081873108928154, | |
| "tokens_seen": 64421888 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.00049518555667001, | |
| "loss": 4.0204, | |
| "theoretical_loss": 5.081112227082771, | |
| "tokens_seen": 64487424 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004951755265797393, | |
| "loss": 3.816, | |
| "theoretical_loss": 5.0803523343591985, | |
| "tokens_seen": 64552960 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004951654964894685, | |
| "loss": 3.8761, | |
| "theoretical_loss": 5.079593428469382, | |
| "tokens_seen": 64618496 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004951554663991976, | |
| "loss": 4.0756, | |
| "theoretical_loss": 5.078835507132873, | |
| "tokens_seen": 64684032 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004951454363089268, | |
| "loss": 3.8136, | |
| "theoretical_loss": 5.078078568076798, | |
| "tokens_seen": 64749568 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000495135406218656, | |
| "loss": 4.0581, | |
| "theoretical_loss": 5.077322609035821, | |
| "tokens_seen": 64815104 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004951253761283852, | |
| "loss": 3.8215, | |
| "theoretical_loss": 5.076567627752118, | |
| "tokens_seen": 64880640 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004951153460381144, | |
| "loss": 3.8355, | |
| "theoretical_loss": 5.075813621975337, | |
| "tokens_seen": 64946176 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004951053159478435, | |
| "loss": 3.9719, | |
| "theoretical_loss": 5.075060589462572, | |
| "tokens_seen": 65011712 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950952858575727, | |
| "loss": 3.7383, | |
| "theoretical_loss": 5.074308527978326, | |
| "tokens_seen": 65077248 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950852557673019, | |
| "loss": 3.8755, | |
| "theoretical_loss": 5.073557435294486, | |
| "tokens_seen": 65142784 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950752256770311, | |
| "loss": 3.6252, | |
| "theoretical_loss": 5.07280730919028, | |
| "tokens_seen": 65208320 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950651955867603, | |
| "loss": 3.7529, | |
| "theoretical_loss": 5.072058147452257, | |
| "tokens_seen": 65273856 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950551654964894, | |
| "loss": 3.679, | |
| "theoretical_loss": 5.071309947874248, | |
| "tokens_seen": 65339392 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950451354062186, | |
| "loss": 3.5894, | |
| "theoretical_loss": 5.070562708257339, | |
| "tokens_seen": 65404928 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950351053159479, | |
| "loss": 3.9951, | |
| "theoretical_loss": 5.069816426409837, | |
| "tokens_seen": 65470464 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "objective/train/docs_used": 48433, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 3.8971009254455566, | |
| "objective/train/theoretical_loss": 5.069071100147241, | |
| "objective/train/tokens_used": 85996000, | |
| "theoretical_loss": 5.069071100147241, | |
| "tokens_seen": 65536000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000495025075225677, | |
| "loss": 3.9133, | |
| "theoretical_loss": 5.069071100147241, | |
| "tokens_seen": 65536000 | |
| } | |
| ], | |
| "max_steps": 50354, | |
| "num_train_epochs": 9223372036854775807, | |
| "total_flos": 3.3445380096e+16, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |