| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.009929697740000794, | |
| "global_step": 500, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.92063492063492e-07, | |
| "loss": 10.8568, | |
| "theoretical_loss": 20.812814784551147, | |
| "tokens_seen": 65536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.984126984126984e-06, | |
| "loss": 10.8727, | |
| "theoretical_loss": 17.56619808733264, | |
| "tokens_seen": 131072 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.9761904761904763e-06, | |
| "loss": 10.7356, | |
| "theoretical_loss": 15.939474075840563, | |
| "tokens_seen": 196608 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.968253968253968e-06, | |
| "loss": 10.7113, | |
| "theoretical_loss": 14.892313738992565, | |
| "tokens_seen": 262144 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.96031746031746e-06, | |
| "loss": 10.3498, | |
| "theoretical_loss": 14.13621392076697, | |
| "tokens_seen": 327680 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.9523809523809525e-06, | |
| "loss": 10.1826, | |
| "theoretical_loss": 13.552558455554218, | |
| "tokens_seen": 393216 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.944444444444444e-06, | |
| "loss": 10.0176, | |
| "theoretical_loss": 13.081805984405184, | |
| "tokens_seen": 458752 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.936507936507936e-06, | |
| "loss": 9.6515, | |
| "theoretical_loss": 12.690126608487319, | |
| "tokens_seen": 524288 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.928571428571428e-06, | |
| "loss": 9.2598, | |
| "theoretical_loss": 12.356589446877619, | |
| "tokens_seen": 589824 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.92063492063492e-06, | |
| "loss": 9.1219, | |
| "theoretical_loss": 12.067409590039073, | |
| "tokens_seen": 655360 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.0912698412698412e-05, | |
| "loss": 9.2934, | |
| "theoretical_loss": 11.813063214105672, | |
| "tokens_seen": 720896 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.1904761904761905e-05, | |
| "loss": 9.0973, | |
| "theoretical_loss": 11.586716191710725, | |
| "tokens_seen": 786432 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.2896825396825396e-05, | |
| "loss": 9.4804, | |
| "theoretical_loss": 11.383311123190783, | |
| "tokens_seen": 851968 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.3888888888888888e-05, | |
| "loss": 9.3208, | |
| "theoretical_loss": 11.199008685115867, | |
| "tokens_seen": 917504 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.4880952380952381e-05, | |
| "loss": 9.1864, | |
| "theoretical_loss": 11.030830900981908, | |
| "tokens_seen": 983040 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.5873015873015872e-05, | |
| "loss": 9.398, | |
| "theoretical_loss": 10.876425069460945, | |
| "tokens_seen": 1048576 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.6865079365079364e-05, | |
| "loss": 8.9394, | |
| "theoretical_loss": 10.73390272306672, | |
| "tokens_seen": 1114112 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.7857142857142855e-05, | |
| "loss": 9.0636, | |
| "theoretical_loss": 10.601726859234274, | |
| "tokens_seen": 1179648 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.884920634920635e-05, | |
| "loss": 9.1583, | |
| "theoretical_loss": 10.478631155360636, | |
| "tokens_seen": 1245184 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.984126984126984e-05, | |
| "loss": 9.1124, | |
| "theoretical_loss": 10.363560926767326, | |
| "tokens_seen": 1310720 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.0833333333333333e-05, | |
| "loss": 9.0087, | |
| "theoretical_loss": 10.25562920390074, | |
| "tokens_seen": 1376256 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.1825396825396824e-05, | |
| "loss": 9.0788, | |
| "theoretical_loss": 10.154083536274014, | |
| "tokens_seen": 1441792 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.2817460317460315e-05, | |
| "loss": 8.4406, | |
| "theoretical_loss": 10.058280544736593, | |
| "tokens_seen": 1507328 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.380952380952381e-05, | |
| "loss": 8.9754, | |
| "theoretical_loss": 9.967666161844274, | |
| "tokens_seen": 1572864 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 12267, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 8.854658126831055, | |
| "objective/train/theoretical_loss": 9.881760109397105, | |
| "objective/train/tokens_used": 22098400, | |
| "theoretical_loss": 9.881760109397105, | |
| "tokens_seen": 1638400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.48015873015873e-05, | |
| "loss": 8.9164, | |
| "theoretical_loss": 9.881760109397105, | |
| "tokens_seen": 1638400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.5793650793650793e-05, | |
| "loss": 7.9452, | |
| "theoretical_loss": 9.800143574544556, | |
| "tokens_seen": 1703936 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.6785714285714284e-05, | |
| "loss": 8.81, | |
| "theoretical_loss": 9.722449329911441, | |
| "tokens_seen": 1769472 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.7777777777777776e-05, | |
| "loss": 9.0741, | |
| "theoretical_loss": 9.648353742085542, | |
| "tokens_seen": 1835008 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.876984126984127e-05, | |
| "loss": 8.8256, | |
| "theoretical_loss": 9.577570254149634, | |
| "tokens_seen": 1900544 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.9761904761904762e-05, | |
| "loss": 8.3274, | |
| "theoretical_loss": 9.509844029768848, | |
| "tokens_seen": 1966080 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.075396825396825e-05, | |
| "loss": 8.55, | |
| "theoretical_loss": 9.44494752063593, | |
| "tokens_seen": 2031616 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.1746031746031745e-05, | |
| "loss": 8.0979, | |
| "theoretical_loss": 9.382676773914453, | |
| "tokens_seen": 2097152 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.273809523809524e-05, | |
| "loss": 8.3374, | |
| "theoretical_loss": 9.322848337237977, | |
| "tokens_seen": 2162688 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.373015873015873e-05, | |
| "loss": 8.5177, | |
| "theoretical_loss": 9.26529664966427, | |
| "tokens_seen": 2228224 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.472222222222222e-05, | |
| "loss": 8.6686, | |
| "theoretical_loss": 9.209871830448751, | |
| "tokens_seen": 2293760 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.571428571428571e-05, | |
| "loss": 8.675, | |
| "theoretical_loss": 9.156437795512288, | |
| "tokens_seen": 2359296 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.670634920634921e-05, | |
| "loss": 8.4694, | |
| "theoretical_loss": 9.104870645417346, | |
| "tokens_seen": 2424832 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.76984126984127e-05, | |
| "loss": 8.2407, | |
| "theoretical_loss": 9.05505727953773, | |
| "tokens_seen": 2490368 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.8690476190476195e-05, | |
| "loss": 8.5934, | |
| "theoretical_loss": 9.006894199647824, | |
| "tokens_seen": 2555904 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 3.968253968253968e-05, | |
| "loss": 7.8491, | |
| "theoretical_loss": 8.960286472913351, | |
| "tokens_seen": 2621440 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.067460317460318e-05, | |
| "loss": 7.8615, | |
| "theoretical_loss": 8.915146829644607, | |
| "tokens_seen": 2686976 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.1666666666666665e-05, | |
| "loss": 7.7589, | |
| "theoretical_loss": 8.871394875482219, | |
| "tokens_seen": 2752512 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.265873015873016e-05, | |
| "loss": 8.0014, | |
| "theoretical_loss": 8.828956401157495, | |
| "tokens_seen": 2818048 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.365079365079365e-05, | |
| "loss": 7.8309, | |
| "theoretical_loss": 8.787762775782408, | |
| "tokens_seen": 2883584 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.464285714285714e-05, | |
| "loss": 8.259, | |
| "theoretical_loss": 8.747750411915451, | |
| "tokens_seen": 2949120 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.563492063492063e-05, | |
| "loss": 8.1107, | |
| "theoretical_loss": 8.708860292524829, | |
| "tokens_seen": 3014656 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.6626984126984126e-05, | |
| "loss": 8.2947, | |
| "theoretical_loss": 8.671037551512843, | |
| "tokens_seen": 3080192 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.761904761904762e-05, | |
| "loss": 7.764, | |
| "theoretical_loss": 8.63423110073947, | |
| "tokens_seen": 3145728 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.8611111111111115e-05, | |
| "loss": 8.1678, | |
| "theoretical_loss": 8.59839329754032, | |
| "tokens_seen": 3211264 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 13483, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 8.108365058898926, | |
| "objective/train/theoretical_loss": 8.563479647615063, | |
| "objective/train/tokens_used": 23736800, | |
| "theoretical_loss": 8.563479647615063, | |
| "tokens_seen": 3276800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.96031746031746e-05, | |
| "loss": 7.5801, | |
| "theoretical_loss": 8.563479647615063, | |
| "tokens_seen": 3276800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.05952380952381e-05, | |
| "loss": 7.8434, | |
| "theoretical_loss": 8.52944853889911, | |
| "tokens_seen": 3342336 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.1587301587301586e-05, | |
| "loss": 7.8492, | |
| "theoretical_loss": 8.496261002649998, | |
| "tokens_seen": 3407872 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.257936507936508e-05, | |
| "loss": 7.8312, | |
| "theoretical_loss": 8.463880498501183, | |
| "tokens_seen": 3473408 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.357142857142857e-05, | |
| "loss": 7.3015, | |
| "theoretical_loss": 8.432272720676774, | |
| "tokens_seen": 3538944 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.4563492063492063e-05, | |
| "loss": 7.9474, | |
| "theoretical_loss": 8.401405422934712, | |
| "tokens_seen": 3604480 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.555555555555555e-05, | |
| "loss": 7.6148, | |
| "theoretical_loss": 8.371248260124204, | |
| "tokens_seen": 3670016 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.6547619047619046e-05, | |
| "loss": 7.5108, | |
| "theoretical_loss": 8.341772644515071, | |
| "tokens_seen": 3735552 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.753968253968254e-05, | |
| "loss": 7.6168, | |
| "theoretical_loss": 8.312951615289325, | |
| "tokens_seen": 3801088 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.8531746031746036e-05, | |
| "loss": 7.8436, | |
| "theoretical_loss": 8.284759719785177, | |
| "tokens_seen": 3866624 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.9523809523809524e-05, | |
| "loss": 7.0651, | |
| "theoretical_loss": 8.257172905255858, | |
| "tokens_seen": 3932160 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.051587301587302e-05, | |
| "loss": 7.4859, | |
| "theoretical_loss": 8.23016842005411, | |
| "tokens_seen": 3997696 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.15079365079365e-05, | |
| "loss": 7.531, | |
| "theoretical_loss": 8.203724723281965, | |
| "tokens_seen": 4063232 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.25e-05, | |
| "loss": 7.5815, | |
| "theoretical_loss": 8.177821402057042, | |
| "tokens_seen": 4128768 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.349206349206349e-05, | |
| "loss": 7.2471, | |
| "theoretical_loss": 8.152439095643611, | |
| "tokens_seen": 4194304 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.448412698412699e-05, | |
| "loss": 7.4488, | |
| "theoretical_loss": 8.127559425781495, | |
| "tokens_seen": 4259840 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.547619047619048e-05, | |
| "loss": 7.4517, | |
| "theoretical_loss": 8.103164932619705, | |
| "tokens_seen": 4325376 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.646825396825397e-05, | |
| "loss": 7.1039, | |
| "theoretical_loss": 8.079239015726635, | |
| "tokens_seen": 4390912 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.746031746031745e-05, | |
| "loss": 7.1279, | |
| "theoretical_loss": 8.055765879705412, | |
| "tokens_seen": 4456448 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.845238095238096e-05, | |
| "loss": 7.3354, | |
| "theoretical_loss": 8.032730483993003, | |
| "tokens_seen": 4521984 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 6.944444444444444e-05, | |
| "loss": 7.1485, | |
| "theoretical_loss": 8.01011849646583, | |
| "tokens_seen": 4587520 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.043650793650793e-05, | |
| "loss": 6.9851, | |
| "theoretical_loss": 7.987916250513374, | |
| "tokens_seen": 4653056 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.142857142857142e-05, | |
| "loss": 6.9432, | |
| "theoretical_loss": 7.966110705275796, | |
| "tokens_seen": 4718592 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.242063492063492e-05, | |
| "loss": 6.9933, | |
| "theoretical_loss": 7.944689408771984, | |
| "tokens_seen": 4784128 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.341269841269842e-05, | |
| "loss": 7.0325, | |
| "theoretical_loss": 7.923640463671514, | |
| "tokens_seen": 4849664 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 14033, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 6.057826519012451, | |
| "objective/train/theoretical_loss": 7.902952495488062, | |
| "objective/train/tokens_used": 25375200, | |
| "theoretical_loss": 7.902952495488062, | |
| "tokens_seen": 4915200 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.440476190476191e-05, | |
| "loss": 6.7791, | |
| "theoretical_loss": 7.902952495488062, | |
| "tokens_seen": 4915200 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.53968253968254e-05, | |
| "loss": 6.7978, | |
| "theoretical_loss": 7.882614622993199, | |
| "tokens_seen": 4980736 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.63888888888889e-05, | |
| "loss": 6.8558, | |
| "theoretical_loss": 7.8626164306686235, | |
| "tokens_seen": 5046272 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.738095238095239e-05, | |
| "loss": 6.9141, | |
| "theoretical_loss": 7.842947943031932, | |
| "tokens_seen": 5111808 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.837301587301588e-05, | |
| "loss": 6.9255, | |
| "theoretical_loss": 7.8235996006863076, | |
| "tokens_seen": 5177344 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.936507936507937e-05, | |
| "loss": 6.7895, | |
| "theoretical_loss": 7.8045622379581605, | |
| "tokens_seen": 5242880 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.035714285714287e-05, | |
| "loss": 6.7059, | |
| "theoretical_loss": 7.785827061999068, | |
| "tokens_seen": 5308416 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.134920634920635e-05, | |
| "loss": 6.815, | |
| "theoretical_loss": 7.767385633239359, | |
| "tokens_seen": 5373952 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.234126984126984e-05, | |
| "loss": 6.8445, | |
| "theoretical_loss": 7.749229847090614, | |
| "tokens_seen": 5439488 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.333333333333333e-05, | |
| "loss": 6.7849, | |
| "theoretical_loss": 7.731351916803314, | |
| "tokens_seen": 5505024 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.432539682539683e-05, | |
| "loss": 6.5349, | |
| "theoretical_loss": 7.7137443573939155, | |
| "tokens_seen": 5570560 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.531746031746032e-05, | |
| "loss": 6.5103, | |
| "theoretical_loss": 7.696399970562929, | |
| "tokens_seen": 5636096 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.630952380952381e-05, | |
| "loss": 6.9307, | |
| "theoretical_loss": 7.679311830532177, | |
| "tokens_seen": 5701632 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.73015873015873e-05, | |
| "loss": 6.8115, | |
| "theoretical_loss": 7.662473270735324, | |
| "tokens_seen": 5767168 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.82936507936508e-05, | |
| "loss": 6.6601, | |
| "theoretical_loss": 7.645877871301275, | |
| "tokens_seen": 5832704 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.928571428571429e-05, | |
| "loss": 6.2238, | |
| "theoretical_loss": 7.6295194472748555, | |
| "tokens_seen": 5898240 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.027777777777777e-05, | |
| "loss": 6.6176, | |
| "theoretical_loss": 7.613392037523692, | |
| "tokens_seen": 5963776 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.126984126984126e-05, | |
| "loss": 6.5932, | |
| "theoretical_loss": 7.597489894284275, | |
| "tokens_seen": 6029312 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.226190476190476e-05, | |
| "loss": 6.51, | |
| "theoretical_loss": 7.581807473303883, | |
| "tokens_seen": 6094848 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.325396825396825e-05, | |
| "loss": 6.3768, | |
| "theoretical_loss": 7.566339424538366, | |
| "tokens_seen": 6160384 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.424603174603175e-05, | |
| "loss": 6.5465, | |
| "theoretical_loss": 7.551080583368945, | |
| "tokens_seen": 6225920 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.523809523809524e-05, | |
| "loss": 6.8931, | |
| "theoretical_loss": 7.536025962303915, | |
| "tokens_seen": 6291456 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.623015873015874e-05, | |
| "loss": 6.5266, | |
| "theoretical_loss": 7.521170743133757, | |
| "tokens_seen": 6356992 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.722222222222223e-05, | |
| "loss": 6.4856, | |
| "theoretical_loss": 7.506510269510493, | |
| "tokens_seen": 6422528 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.821428571428572e-05, | |
| "loss": 6.4763, | |
| "theoretical_loss": 7.492040039924245, | |
| "tokens_seen": 6488064 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 15357, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 6.699707984924316, | |
| "objective/train/theoretical_loss": 7.4777557010520255, | |
| "objective/train/tokens_used": 27013600, | |
| "theoretical_loss": 7.4777557010520255, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.92063492063492e-05, | |
| "loss": 6.6532, | |
| "theoretical_loss": 7.4777557010520255, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010019841269841271, | |
| "loss": 6.4, | |
| "theoretical_loss": 7.463653041455457, | |
| "tokens_seen": 6619136 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001011904761904762, | |
| "loss": 6.2462, | |
| "theoretical_loss": 7.449727985605911, | |
| "tokens_seen": 6684672 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010218253968253968, | |
| "loss": 6.6399, | |
| "theoretical_loss": 7.435976588217015, | |
| "tokens_seen": 6750208 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010317460317460317, | |
| "loss": 6.1887, | |
| "theoretical_loss": 7.4223950288659, | |
| "tokens_seen": 6815744 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010416666666666667, | |
| "loss": 6.4129, | |
| "theoretical_loss": 7.40897960688587, | |
| "tokens_seen": 6881280 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010515873015873016, | |
| "loss": 6.2296, | |
| "theoretical_loss": 7.395726736514341, | |
| "tokens_seen": 6946816 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010615079365079365, | |
| "loss": 6.3505, | |
| "theoretical_loss": 7.3826329422810275, | |
| "tokens_seen": 7012352 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010714285714285714, | |
| "loss": 6.5043, | |
| "theoretical_loss": 7.369694854622368, | |
| "tokens_seen": 7077888 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010813492063492064, | |
| "loss": 6.3689, | |
| "theoretical_loss": 7.3569092057090835, | |
| "tokens_seen": 7143424 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00010912698412698413, | |
| "loss": 6.5583, | |
| "theoretical_loss": 7.3442728254746825, | |
| "tokens_seen": 7208960 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011011904761904761, | |
| "loss": 6.3757, | |
| "theoretical_loss": 7.3317826378335145, | |
| "tokens_seen": 7274496 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001111111111111111, | |
| "loss": 6.3164, | |
| "theoretical_loss": 7.319435657077673, | |
| "tokens_seen": 7340032 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001121031746031746, | |
| "loss": 6.205, | |
| "theoretical_loss": 7.307228984442819, | |
| "tokens_seen": 7405568 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011309523809523809, | |
| "loss": 6.4975, | |
| "theoretical_loss": 7.29515980483356, | |
| "tokens_seen": 7471104 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011408730158730158, | |
| "loss": 6.087, | |
| "theoretical_loss": 7.283225383699648, | |
| "tokens_seen": 7536640 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011507936507936508, | |
| "loss": 6.0785, | |
| "theoretical_loss": 7.271423064054828, | |
| "tokens_seen": 7602176 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011607142857142858, | |
| "loss": 6.4949, | |
| "theoretical_loss": 7.259750263630618, | |
| "tokens_seen": 7667712 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011706349206349207, | |
| "loss": 6.2155, | |
| "theoretical_loss": 7.248204472157866, | |
| "tokens_seen": 7733248 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011805555555555556, | |
| "loss": 6.0229, | |
| "theoretical_loss": 7.236783248769257, | |
| "tokens_seen": 7798784 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011904761904761905, | |
| "loss": 6.6244, | |
| "theoretical_loss": 7.225484219516492, | |
| "tokens_seen": 7864320 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012003968253968255, | |
| "loss": 5.9387, | |
| "theoretical_loss": 7.2143050749961155, | |
| "tokens_seen": 7929856 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012103174603174604, | |
| "loss": 6.2842, | |
| "theoretical_loss": 7.2032435680783955, | |
| "tokens_seen": 7995392 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012202380952380953, | |
| "loss": 6.2047, | |
| "theoretical_loss": 7.19229751173401, | |
| "tokens_seen": 8060928 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.000123015873015873, | |
| "loss": 6.2571, | |
| "theoretical_loss": 7.181464776953511, | |
| "tokens_seen": 8126464 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 16711, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 6.2001729011535645, | |
| "objective/train/theoretical_loss": 7.1707432907549595, | |
| "objective/train/tokens_used": 28652000, | |
| "theoretical_loss": 7.1707432907549595, | |
| "tokens_seen": 8192000 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001240079365079365, | |
| "loss": 6.0533, | |
| "theoretical_loss": 7.1707432907549595, | |
| "tokens_seen": 8192000 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.000125, | |
| "loss": 6.0526, | |
| "theoretical_loss": 7.160131034275267, | |
| "tokens_seen": 8257536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001259920634920635, | |
| "loss": 6.0532, | |
| "theoretical_loss": 7.149626040941133, | |
| "tokens_seen": 8323072 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012698412698412698, | |
| "loss": 5.9719, | |
| "theoretical_loss": 7.139226394715633, | |
| "tokens_seen": 8388608 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012797619047619048, | |
| "loss": 6.0403, | |
| "theoretical_loss": 7.12893022841679, | |
| "tokens_seen": 8454144 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012896825396825398, | |
| "loss": 5.7847, | |
| "theoretical_loss": 7.118735722104612, | |
| "tokens_seen": 8519680 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00012996031746031748, | |
| "loss": 5.9094, | |
| "theoretical_loss": 7.10864110153334, | |
| "tokens_seen": 8585216 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00013095238095238096, | |
| "loss": 6.0872, | |
| "theoretical_loss": 7.098644636665766, | |
| "tokens_seen": 8650752 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00013194444444444446, | |
| "loss": 6.0185, | |
| "theoretical_loss": 7.088744640246688, | |
| "tokens_seen": 8716288 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00013293650793650793, | |
| "loss": 6.0768, | |
| "theoretical_loss": 7.078939466432745, | |
| "tokens_seen": 8781824 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00013392857142857144, | |
| "loss": 6.3022, | |
| "theoretical_loss": 7.069227509475962, | |
| "tokens_seen": 8847360 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001349206349206349, | |
| "loss": 6.0881, | |
| "theoretical_loss": 7.059607202458563, | |
| "tokens_seen": 8912896 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001359126984126984, | |
| "loss": 5.7381, | |
| "theoretical_loss": 7.0500770160766635, | |
| "tokens_seen": 8978432 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001369047619047619, | |
| "loss": 5.9916, | |
| "theoretical_loss": 7.04063545747062, | |
| "tokens_seen": 9043968 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00013789682539682541, | |
| "loss": 6.0983, | |
| "theoretical_loss": 7.031281069099929, | |
| "tokens_seen": 9109504 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001388888888888889, | |
| "loss": 6.0662, | |
| "theoretical_loss": 7.022012427660673, | |
| "tokens_seen": 9175040 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001398809523809524, | |
| "loss": 5.7078, | |
| "theoretical_loss": 7.012828143043604, | |
| "tokens_seen": 9240576 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014087301587301586, | |
| "loss": 6.2455, | |
| "theoretical_loss": 7.003726857331067, | |
| "tokens_seen": 9306112 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014186507936507937, | |
| "loss": 5.9609, | |
| "theoretical_loss": 6.994707243831053, | |
| "tokens_seen": 9371648 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014285714285714284, | |
| "loss": 5.9423, | |
| "theoretical_loss": 6.985768006146776, | |
| "tokens_seen": 9437184 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014384920634920634, | |
| "loss": 6.0329, | |
| "theoretical_loss": 6.976907877280185, | |
| "tokens_seen": 9502720 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014484126984126984, | |
| "loss": 5.7799, | |
| "theoretical_loss": 6.96812561876801, | |
| "tokens_seen": 9568256 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014583333333333335, | |
| "loss": 5.9063, | |
| "theoretical_loss": 6.95942001984889, | |
| "tokens_seen": 9633792 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014682539682539685, | |
| "loss": 5.9317, | |
| "theoretical_loss": 6.950789896660304, | |
| "tokens_seen": 9699328 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014781746031746032, | |
| "loss": 5.8925, | |
| "theoretical_loss": 6.942234091464025, | |
| "tokens_seen": 9764864 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 17166, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 6.479184627532959, | |
| "objective/train/theoretical_loss": 6.933751471898896, | |
| "objective/train/tokens_used": 30290400, | |
| "theoretical_loss": 6.933751471898896, | |
| "tokens_seen": 9830400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014880952380952382, | |
| "loss": 6.2539, | |
| "theoretical_loss": 6.933751471898896, | |
| "tokens_seen": 9830400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001498015873015873, | |
| "loss": 5.996, | |
| "theoretical_loss": 6.925340930259813, | |
| "tokens_seen": 9895936 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001507936507936508, | |
| "loss": 5.8197, | |
| "theoretical_loss": 6.917001382801793, | |
| "tokens_seen": 9961472 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015178571428571427, | |
| "loss": 6.0534, | |
| "theoretical_loss": 6.908731769068142, | |
| "tokens_seen": 10027008 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001527777777777778, | |
| "loss": 5.7838, | |
| "theoretical_loss": 6.900531051241684, | |
| "tokens_seen": 10092544 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015376984126984128, | |
| "loss": 5.7799, | |
| "theoretical_loss": 6.892398213518156, | |
| "tokens_seen": 10158080 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015476190476190478, | |
| "loss": 6.1056, | |
| "theoretical_loss": 6.884332261500866, | |
| "tokens_seen": 10223616 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015575396825396825, | |
| "loss": 5.5432, | |
| "theoretical_loss": 6.876332221615746, | |
| "tokens_seen": 10289152 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015674603174603175, | |
| "loss": 5.8572, | |
| "theoretical_loss": 6.8683971405459925, | |
| "tokens_seen": 10354688 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015773809523809523, | |
| "loss": 6.226, | |
| "theoretical_loss": 6.860526084685546, | |
| "tokens_seen": 10420224 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00015873015873015873, | |
| "loss": 5.7117, | |
| "theoretical_loss": 6.852718139610625, | |
| "tokens_seen": 10485760 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001597222222222222, | |
| "loss": 5.6684, | |
| "theoretical_loss": 6.844972409568637, | |
| "tokens_seen": 10551296 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016071428571428573, | |
| "loss": 5.8166, | |
| "theoretical_loss": 6.8372880169837895, | |
| "tokens_seen": 10616832 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001617063492063492, | |
| "loss": 5.971, | |
| "theoretical_loss": 6.829664101978745, | |
| "tokens_seen": 10682368 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001626984126984127, | |
| "loss": 5.7343, | |
| "theoretical_loss": 6.8220998219117135, | |
| "tokens_seen": 10747904 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016369047619047618, | |
| "loss": 5.8061, | |
| "theoretical_loss": 6.81459435092839, | |
| "tokens_seen": 10813440 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016468253968253969, | |
| "loss": 6.0029, | |
| "theoretical_loss": 6.807146879528176, | |
| "tokens_seen": 10878976 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016567460317460316, | |
| "loss": 5.5367, | |
| "theoretical_loss": 6.79975661414414, | |
| "tokens_seen": 10944512 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016666666666666666, | |
| "loss": 5.6256, | |
| "theoretical_loss": 6.79242277673622, | |
| "tokens_seen": 11010048 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016765873015873016, | |
| "loss": 5.7868, | |
| "theoretical_loss": 6.785144604397143, | |
| "tokens_seen": 11075584 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016865079365079366, | |
| "loss": 5.6046, | |
| "theoretical_loss": 6.777921348970633, | |
| "tokens_seen": 11141120 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016964285714285717, | |
| "loss": 5.9753, | |
| "theoretical_loss": 6.770752276681419, | |
| "tokens_seen": 11206656 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017063492063492064, | |
| "loss": 5.9022, | |
| "theoretical_loss": 6.76363666777662, | |
| "tokens_seen": 11272192 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017162698412698414, | |
| "loss": 5.8921, | |
| "theoretical_loss": 6.756573816178118, | |
| "tokens_seen": 11337728 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017261904761904762, | |
| "loss": 5.9722, | |
| "theoretical_loss": 6.749563029145481, | |
| "tokens_seen": 11403264 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 18369, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.672413349151611, | |
| "objective/train/theoretical_loss": 6.742603626949085, | |
| "objective/train/tokens_used": 31928800, | |
| "theoretical_loss": 6.742603626949085, | |
| "tokens_seen": 11468800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017361111111111112, | |
| "loss": 5.4506, | |
| "theoretical_loss": 6.742603626949085, | |
| "tokens_seen": 11468800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001746031746031746, | |
| "loss": 6.0078, | |
| "theoretical_loss": 6.735694942553071, | |
| "tokens_seen": 11534336 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001755952380952381, | |
| "loss": 5.8091, | |
| "theoretical_loss": 6.728836321307757, | |
| "tokens_seen": 11599872 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001765873015873016, | |
| "loss": 5.6113, | |
| "theoretical_loss": 6.722027120651221, | |
| "tokens_seen": 11665408 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001775793650793651, | |
| "loss": 5.6758, | |
| "theoretical_loss": 6.715266709819684, | |
| "tokens_seen": 11730944 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017857142857142857, | |
| "loss": 5.6378, | |
| "theoretical_loss": 6.708554469566408, | |
| "tokens_seen": 11796480 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017956349206349207, | |
| "loss": 5.7411, | |
| "theoretical_loss": 6.701889791888819, | |
| "tokens_seen": 11862016 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018055555555555555, | |
| "loss": 5.8311, | |
| "theoretical_loss": 6.695272079763555, | |
| "tokens_seen": 11927552 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018154761904761905, | |
| "loss": 5.2846, | |
| "theoretical_loss": 6.688700746889175, | |
| "tokens_seen": 11993088 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018253968253968252, | |
| "loss": 5.7687, | |
| "theoretical_loss": 6.682175217436269, | |
| "tokens_seen": 12058624 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018353174603174602, | |
| "loss": 5.7687, | |
| "theoretical_loss": 6.675694925804711, | |
| "tokens_seen": 12124160 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018452380952380953, | |
| "loss": 5.6209, | |
| "theoretical_loss": 6.66925931638781, | |
| "tokens_seen": 12189696 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018551587301587303, | |
| "loss": 5.9193, | |
| "theoretical_loss": 6.662867843343154, | |
| "tokens_seen": 12255232 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001865079365079365, | |
| "loss": 5.6754, | |
| "theoretical_loss": 6.6565199703698745, | |
| "tokens_seen": 12320768 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001875, | |
| "loss": 5.8597, | |
| "theoretical_loss": 6.6502151704921655, | |
| "tokens_seen": 12386304 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001884920634920635, | |
| "loss": 5.709, | |
| "theoretical_loss": 6.643952925848826, | |
| "tokens_seen": 12451840 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018948412698412698, | |
| "loss": 5.7054, | |
| "theoretical_loss": 6.637732727488622, | |
| "tokens_seen": 12517376 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019047619047619048, | |
| "loss": 5.5972, | |
| "theoretical_loss": 6.6315540751713, | |
| "tokens_seen": 12582912 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019146825396825398, | |
| "loss": 5.5887, | |
| "theoretical_loss": 6.625416477174044, | |
| "tokens_seen": 12648448 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019246031746031748, | |
| "loss": 5.7082, | |
| "theoretical_loss": 6.619319450103218, | |
| "tokens_seen": 12713984 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019345238095238096, | |
| "loss": 5.5405, | |
| "theoretical_loss": 6.613262518711206, | |
| "tokens_seen": 12779520 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019444444444444446, | |
| "loss": 5.5806, | |
| "theoretical_loss": 6.607245215718208, | |
| "tokens_seen": 12845056 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019543650793650793, | |
| "loss": 5.4612, | |
| "theoretical_loss": 6.601267081638806, | |
| "tokens_seen": 12910592 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019642857142857144, | |
| "loss": 5.6405, | |
| "theoretical_loss": 6.595327664613176, | |
| "tokens_seen": 12976128 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001974206349206349, | |
| "loss": 5.3968, | |
| "theoretical_loss": 6.58942652024278, | |
| "tokens_seen": 13041664 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 19017, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.398904323577881, | |
| "objective/train/theoretical_loss": 6.583563211430409, | |
| "objective/train/tokens_used": 33567200, | |
| "theoretical_loss": 6.583563211430409, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001984126984126984, | |
| "loss": 5.5392, | |
| "theoretical_loss": 6.583563211430409, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00019940476190476191, | |
| "loss": 5.3995, | |
| "theoretical_loss": 6.577737308224426, | |
| "tokens_seen": 13172736 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020039682539682542, | |
| "loss": 5.5665, | |
| "theoretical_loss": 6.571948387667094, | |
| "tokens_seen": 13238272 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002013888888888889, | |
| "loss": 5.5745, | |
| "theoretical_loss": 6.566196033646859, | |
| "tokens_seen": 13303808 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002023809523809524, | |
| "loss": 5.4756, | |
| "theoretical_loss": 6.560479836754459, | |
| "tokens_seen": 13369344 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020337301587301587, | |
| "loss": 5.7895, | |
| "theoretical_loss": 6.554799394142741, | |
| "tokens_seen": 13434880 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020436507936507937, | |
| "loss": 5.3965, | |
| "theoretical_loss": 6.549154309390087, | |
| "tokens_seen": 13500416 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020535714285714284, | |
| "loss": 5.494, | |
| "theoretical_loss": 6.5435441923673245, | |
| "tokens_seen": 13565952 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020634920634920634, | |
| "loss": 5.5559, | |
| "theoretical_loss": 6.537968659108021, | |
| "tokens_seen": 13631488 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020734126984126985, | |
| "loss": 5.9125, | |
| "theoretical_loss": 6.532427331682063, | |
| "tokens_seen": 13697024 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020833333333333335, | |
| "loss": 5.5235, | |
| "theoretical_loss": 6.5269198380724145, | |
| "tokens_seen": 13762560 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00020932539682539685, | |
| "loss": 5.6307, | |
| "theoretical_loss": 6.5214458120549725, | |
| "tokens_seen": 13828096 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021031746031746032, | |
| "loss": 5.3661, | |
| "theoretical_loss": 6.516004893081412, | |
| "tokens_seen": 13893632 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021130952380952382, | |
| "loss": 5.2823, | |
| "theoretical_loss": 6.510596726164945, | |
| "tokens_seen": 13959168 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002123015873015873, | |
| "loss": 5.4241, | |
| "theoretical_loss": 6.505220961768906, | |
| "tokens_seen": 14024704 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002132936507936508, | |
| "loss": 5.3679, | |
| "theoretical_loss": 6.499877255698063, | |
| "tokens_seen": 14090240 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021428571428571427, | |
| "loss": 5.2005, | |
| "theoretical_loss": 6.494565268992613, | |
| "tokens_seen": 14155776 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002152777777777778, | |
| "loss": 5.6155, | |
| "theoretical_loss": 6.489284667824741, | |
| "tokens_seen": 14221312 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021626984126984128, | |
| "loss": 5.6027, | |
| "theoretical_loss": 6.484035123397694, | |
| "tokens_seen": 14286848 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021726190476190478, | |
| "loss": 5.5959, | |
| "theoretical_loss": 6.478816311847305, | |
| "tokens_seen": 14352384 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021825396825396825, | |
| "loss": 5.7842, | |
| "theoretical_loss": 6.473627914145865, | |
| "tokens_seen": 14417920 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00021924603174603176, | |
| "loss": 5.4174, | |
| "theoretical_loss": 6.468469616008303, | |
| "tokens_seen": 14483456 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022023809523809523, | |
| "loss": 5.4423, | |
| "theoretical_loss": 6.463341107800611, | |
| "tokens_seen": 14548992 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022123015873015873, | |
| "loss": 5.2413, | |
| "theoretical_loss": 6.458242084450426, | |
| "tokens_seen": 14614528 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002222222222222222, | |
| "loss": 5.4841, | |
| "theoretical_loss": 6.453172245359726, | |
| "tokens_seen": 14680064 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 20187, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.4210638999938965, | |
| "objective/train/theoretical_loss": 6.448131294319589, | |
| "objective/train/tokens_used": 35205600, | |
| "theoretical_loss": 6.448131294319589, | |
| "tokens_seen": 14745600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022321428571428573, | |
| "loss": 5.318, | |
| "theoretical_loss": 6.448131294319589, | |
| "tokens_seen": 14745600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002242063492063492, | |
| "loss": 5.5298, | |
| "theoretical_loss": 6.443118939426935, | |
| "tokens_seen": 14811136 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002251984126984127, | |
| "loss": 5.5104, | |
| "theoretical_loss": 6.438134893003209, | |
| "tokens_seen": 14876672 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022619047619047618, | |
| "loss": 5.4089, | |
| "theoretical_loss": 6.43317887151496, | |
| "tokens_seen": 14942208 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022718253968253969, | |
| "loss": 5.5783, | |
| "theoretical_loss": 6.428250595496234, | |
| "tokens_seen": 15007744 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022817460317460316, | |
| "loss": 5.4029, | |
| "theoretical_loss": 6.4233497894727805, | |
| "tokens_seen": 15073280 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00022916666666666666, | |
| "loss": 5.6043, | |
| "theoretical_loss": 6.418476181887964, | |
| "tokens_seen": 15138816 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023015873015873016, | |
| "loss": 5.5123, | |
| "theoretical_loss": 6.413629505030386, | |
| "tokens_seen": 15204352 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023115079365079367, | |
| "loss": 5.5084, | |
| "theoretical_loss": 6.408809494963135, | |
| "tokens_seen": 15269888 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023214285714285717, | |
| "loss": 5.269, | |
| "theoretical_loss": 6.404015891454651, | |
| "tokens_seen": 15335424 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023313492063492064, | |
| "loss": 5.4379, | |
| "theoretical_loss": 6.399248437911128, | |
| "tokens_seen": 15400960 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023412698412698414, | |
| "loss": 5.4346, | |
| "theoretical_loss": 6.394506881310447, | |
| "tokens_seen": 15466496 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023511904761904762, | |
| "loss": 5.7191, | |
| "theoretical_loss": 6.38979097213757, | |
| "tokens_seen": 15532032 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023611111111111112, | |
| "loss": 5.5484, | |
| "theoretical_loss": 6.385100464321383, | |
| "tokens_seen": 15597568 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002371031746031746, | |
| "loss": 5.4379, | |
| "theoretical_loss": 6.380435115172919, | |
| "tokens_seen": 15663104 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002380952380952381, | |
| "loss": 5.4785, | |
| "theoretical_loss": 6.375794685324962, | |
| "tokens_seen": 15728640 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002390873015873016, | |
| "loss": 5.2088, | |
| "theoretical_loss": 6.371178938672962, | |
| "tokens_seen": 15794176 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002400793650793651, | |
| "loss": 5.5076, | |
| "theoretical_loss": 6.366587642317243, | |
| "tokens_seen": 15859712 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024107142857142857, | |
| "loss": 5.6351, | |
| "theoretical_loss": 6.362020566506475, | |
| "tokens_seen": 15925248 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024206349206349207, | |
| "loss": 5.6029, | |
| "theoretical_loss": 6.3574774845823665, | |
| "tokens_seen": 15990784 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024305555555555555, | |
| "loss": 5.1969, | |
| "theoretical_loss": 6.352958172925549, | |
| "tokens_seen": 16056320 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024404761904761905, | |
| "loss": 5.285, | |
| "theoretical_loss": 6.348462410902625, | |
| "tokens_seen": 16121856 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024503968253968255, | |
| "loss": 5.6805, | |
| "theoretical_loss": 6.343989980814362, | |
| "tokens_seen": 16187392 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.000246031746031746, | |
| "loss": 5.5855, | |
| "theoretical_loss": 6.339540667844965, | |
| "tokens_seen": 16252928 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00024702380952380955, | |
| "loss": 5.3559, | |
| "theoretical_loss": 6.335114260012455, | |
| "tokens_seen": 16318464 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "objective/train/docs_used": 20737, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.463630199432373, | |
| "objective/train/theoretical_loss": 6.330710548120079, | |
| "objective/train/tokens_used": 36844000, | |
| "theoretical_loss": 6.330710548120079, | |
| "tokens_seen": 16384000 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.000248015873015873, | |
| "loss": 5.515, | |
| "theoretical_loss": 6.330710548120079, | |
| "tokens_seen": 16384000 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002490079365079365, | |
| "loss": 5.3596, | |
| "theoretical_loss": 6.326329325708746, | |
| "tokens_seen": 16449536 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025, | |
| "loss": 5.249, | |
| "theoretical_loss": 6.321970389010465, | |
| "tokens_seen": 16515072 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002509920634920635, | |
| "loss": 5.3675, | |
| "theoretical_loss": 6.317633536902759, | |
| "tokens_seen": 16580608 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000251984126984127, | |
| "loss": 5.0919, | |
| "theoretical_loss": 6.313318570864016, | |
| "tokens_seen": 16646144 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025297619047619046, | |
| "loss": 5.1228, | |
| "theoretical_loss": 6.30902529492978, | |
| "tokens_seen": 16711680 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025396825396825396, | |
| "loss": 5.4573, | |
| "theoretical_loss": 6.304753515649935, | |
| "tokens_seen": 16777216 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025496031746031746, | |
| "loss": 5.1499, | |
| "theoretical_loss": 6.30050304204677, | |
| "tokens_seen": 16842752 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025595238095238096, | |
| "loss": 5.6432, | |
| "theoretical_loss": 6.296273685573913, | |
| "tokens_seen": 16908288 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002569444444444444, | |
| "loss": 5.439, | |
| "theoretical_loss": 6.292065260076094, | |
| "tokens_seen": 16973824 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025793650793650796, | |
| "loss": 5.2959, | |
| "theoretical_loss": 6.287877581749726, | |
| "tokens_seen": 17039360 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025892857142857146, | |
| "loss": 5.3502, | |
| "theoretical_loss": 6.2837104691042915, | |
| "tokens_seen": 17104896 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00025992063492063497, | |
| "loss": 5.5134, | |
| "theoretical_loss": 6.279563742924502, | |
| "tokens_seen": 17170432 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002609126984126984, | |
| "loss": 5.4871, | |
| "theoretical_loss": 6.275437226233224, | |
| "tokens_seen": 17235968 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002619047619047619, | |
| "loss": 5.3132, | |
| "theoretical_loss": 6.271330744255137, | |
| "tokens_seen": 17301504 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002628968253968254, | |
| "loss": 5.1796, | |
| "theoretical_loss": 6.267244124381133, | |
| "tokens_seen": 17367040 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002638888888888889, | |
| "loss": 5.2656, | |
| "theoretical_loss": 6.2631771961334035, | |
| "tokens_seen": 17432576 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00026488095238095237, | |
| "loss": 5.5896, | |
| "theoretical_loss": 6.259129791131242, | |
| "tokens_seen": 17498112 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00026587301587301587, | |
| "loss": 5.14, | |
| "theoretical_loss": 6.255101743057493, | |
| "tokens_seen": 17563648 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00026686507936507937, | |
| "loss": 5.3606, | |
| "theoretical_loss": 6.251092887625685, | |
| "tokens_seen": 17629184 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00026785714285714287, | |
| "loss": 5.487, | |
| "theoretical_loss": 6.247103062547796, | |
| "tokens_seen": 17694720 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002688492063492063, | |
| "loss": 5.2887, | |
| "theoretical_loss": 6.243132107502647, | |
| "tokens_seen": 17760256 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002698412698412698, | |
| "loss": 5.4254, | |
| "theoretical_loss": 6.239179864104911, | |
| "tokens_seen": 17825792 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002708333333333333, | |
| "loss": 5.4668, | |
| "theoretical_loss": 6.235246175874727, | |
| "tokens_seen": 17891328 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002718253968253968, | |
| "loss": 5.3584, | |
| "theoretical_loss": 6.231330888207894, | |
| "tokens_seen": 17956864 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 21782, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.119491100311279, | |
| "objective/train/theoretical_loss": 6.227433848346639, | |
| "objective/train/tokens_used": 38482400, | |
| "theoretical_loss": 6.227433848346639, | |
| "tokens_seen": 18022400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002728174603174603, | |
| "loss": 5.1069, | |
| "theoretical_loss": 6.227433848346639, | |
| "tokens_seen": 18022400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002738095238095238, | |
| "loss": 5.1568, | |
| "theoretical_loss": 6.22355490535095, | |
| "tokens_seen": 18087936 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002748015873015873, | |
| "loss": 5.4409, | |
| "theoretical_loss": 6.219693910070452, | |
| "tokens_seen": 18153472 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00027579365079365083, | |
| "loss": 5.1939, | |
| "theoretical_loss": 6.215850715116817, | |
| "tokens_seen": 18219008 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00027678571428571433, | |
| "loss": 5.2274, | |
| "theoretical_loss": 6.212025174836697, | |
| "tokens_seen": 18284544 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002777777777777778, | |
| "loss": 5.5299, | |
| "theoretical_loss": 6.208217145285173, | |
| "tokens_seen": 18350080 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002787698412698413, | |
| "loss": 5.4441, | |
| "theoretical_loss": 6.204426484199696, | |
| "tokens_seen": 18415616 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002797619047619048, | |
| "loss": 5.3135, | |
| "theoretical_loss": 6.200653050974525, | |
| "tokens_seen": 18481152 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002807539682539683, | |
| "loss": 5.189, | |
| "theoretical_loss": 6.196896706635635, | |
| "tokens_seen": 18546688 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00028174603174603173, | |
| "loss": 5.0786, | |
| "theoretical_loss": 6.1931573138160925, | |
| "tokens_seen": 18612224 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00028273809523809523, | |
| "loss": 5.0434, | |
| "theoretical_loss": 6.189434736731897, | |
| "tokens_seen": 18677760 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00028373015873015873, | |
| "loss": 5.0132, | |
| "theoretical_loss": 6.185728841158257, | |
| "tokens_seen": 18743296 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00028472222222222223, | |
| "loss": 5.1159, | |
| "theoretical_loss": 6.182039494406309, | |
| "tokens_seen": 18808832 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002857142857142857, | |
| "loss": 5.1786, | |
| "theoretical_loss": 6.178366565300266, | |
| "tokens_seen": 18874368 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002867063492063492, | |
| "loss": 5.1947, | |
| "theoretical_loss": 6.174709924154978, | |
| "tokens_seen": 18939904 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002876984126984127, | |
| "loss": 5.2396, | |
| "theoretical_loss": 6.171069442753909, | |
| "tokens_seen": 19005440 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002886904761904762, | |
| "loss": 5.3947, | |
| "theoretical_loss": 6.1674449943275045, | |
| "tokens_seen": 19070976 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002896825396825397, | |
| "loss": 4.8732, | |
| "theoretical_loss": 6.16383645353196, | |
| "tokens_seen": 19136512 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002906746031746032, | |
| "loss": 5.3178, | |
| "theoretical_loss": 6.160243696428367, | |
| "tokens_seen": 19202048 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002916666666666667, | |
| "loss": 5.1441, | |
| "theoretical_loss": 6.156666600462238, | |
| "tokens_seen": 19267584 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002926587301587302, | |
| "loss": 5.2695, | |
| "theoretical_loss": 6.153105044443393, | |
| "tokens_seen": 19333120 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002936507936507937, | |
| "loss": 5.3181, | |
| "theoretical_loss": 6.149558908526206, | |
| "tokens_seen": 19398656 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029464285714285714, | |
| "loss": 5.2188, | |
| "theoretical_loss": 6.146028074190217, | |
| "tokens_seen": 19464192 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029563492063492064, | |
| "loss": 5.1652, | |
| "theoretical_loss": 6.14251242422106, | |
| "tokens_seen": 19529728 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029662698412698414, | |
| "loss": 5.3475, | |
| "theoretical_loss": 6.139011842691756, | |
| "tokens_seen": 19595264 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 22379, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.661308765411377, | |
| "objective/train/theoretical_loss": 6.135526214944321, | |
| "objective/train/tokens_used": 40120800, | |
| "theoretical_loss": 6.135526214944321, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029761904761904765, | |
| "loss": 5.6218, | |
| "theoretical_loss": 6.135526214944321, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002986111111111111, | |
| "loss": 5.4528, | |
| "theoretical_loss": 6.1320554275717, | |
| "tokens_seen": 19726336 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002996031746031746, | |
| "loss": 5.1971, | |
| "theoretical_loss": 6.128599368400017, | |
| "tokens_seen": 19791872 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003005952380952381, | |
| "loss": 5.2823, | |
| "theoretical_loss": 6.125157926471134, | |
| "tokens_seen": 19857408 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003015873015873016, | |
| "loss": 5.1154, | |
| "theoretical_loss": 6.121730992025516, | |
| "tokens_seen": 19922944 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030257936507936505, | |
| "loss": 4.8293, | |
| "theoretical_loss": 6.118318456485394, | |
| "tokens_seen": 19988480 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030357142857142855, | |
| "loss": 5.0591, | |
| "theoretical_loss": 6.114920212438209, | |
| "tokens_seen": 20054016 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030456349206349205, | |
| "loss": 5.2452, | |
| "theoretical_loss": 6.111536153620355, | |
| "tokens_seen": 20119552 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003055555555555556, | |
| "loss": 5.1158, | |
| "theoretical_loss": 6.108166174901191, | |
| "tokens_seen": 20185088 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030654761904761905, | |
| "loss": 5.0927, | |
| "theoretical_loss": 6.104810172267331, | |
| "tokens_seen": 20250624 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030753968253968255, | |
| "loss": 5.1097, | |
| "theoretical_loss": 6.101468042807199, | |
| "tokens_seen": 20316160 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030853174603174605, | |
| "loss": 5.0109, | |
| "theoretical_loss": 6.098139684695851, | |
| "tokens_seen": 20381696 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030952380952380956, | |
| "loss": 5.2, | |
| "theoretical_loss": 6.094824997180048, | |
| "tokens_seen": 20447232 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000310515873015873, | |
| "loss": 4.919, | |
| "theoretical_loss": 6.091523880563589, | |
| "tokens_seen": 20512768 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003115079365079365, | |
| "loss": 4.7726, | |
| "theoretical_loss": 6.088236236192881, | |
| "tokens_seen": 20578304 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003125, | |
| "loss": 5.1579, | |
| "theoretical_loss": 6.0849619664427586, | |
| "tokens_seen": 20643840 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003134920634920635, | |
| "loss": 5.2649, | |
| "theoretical_loss": 6.0817009747025415, | |
| "tokens_seen": 20709376 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000314484126984127, | |
| "loss": 4.9338, | |
| "theoretical_loss": 6.07845316536232, | |
| "tokens_seen": 20774912 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00031547619047619046, | |
| "loss": 5.2449, | |
| "theoretical_loss": 6.075218443799468, | |
| "tokens_seen": 20840448 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00031646825396825396, | |
| "loss": 5.2222, | |
| "theoretical_loss": 6.071996716365382, | |
| "tokens_seen": 20905984 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00031746031746031746, | |
| "loss": 4.8122, | |
| "theoretical_loss": 6.068787890372443, | |
| "tokens_seen": 20971520 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00031845238095238096, | |
| "loss": 4.983, | |
| "theoretical_loss": 6.0655918740811865, | |
| "tokens_seen": 21037056 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003194444444444444, | |
| "loss": 4.9428, | |
| "theoretical_loss": 6.062408576687682, | |
| "tokens_seen": 21102592 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032043650793650796, | |
| "loss": 5.0493, | |
| "theoretical_loss": 6.059237908311129, | |
| "tokens_seen": 21168128 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032142857142857147, | |
| "loss": 4.8675, | |
| "theoretical_loss": 6.056079779981644, | |
| "tokens_seen": 21233664 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 23601, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.199455738067627, | |
| "objective/train/theoretical_loss": 6.052934103628253, | |
| "objective/train/tokens_used": 41759200, | |
| "theoretical_loss": 6.052934103628253, | |
| "tokens_seen": 21299200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032242063492063497, | |
| "loss": 5.2637, | |
| "theoretical_loss": 6.052934103628253, | |
| "tokens_seen": 21299200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003234126984126984, | |
| "loss": 5.4325, | |
| "theoretical_loss": 6.049800792067078, | |
| "tokens_seen": 21364736 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003244047619047619, | |
| "loss": 4.8274, | |
| "theoretical_loss": 6.046679758989709, | |
| "tokens_seen": 21430272 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003253968253968254, | |
| "loss": 5.2104, | |
| "theoretical_loss": 6.043570918951775, | |
| "tokens_seen": 21495808 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003263888888888889, | |
| "loss": 5.1036, | |
| "theoretical_loss": 6.040474187361681, | |
| "tokens_seen": 21561344 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032738095238095237, | |
| "loss": 4.8811, | |
| "theoretical_loss": 6.037389480469548, | |
| "tokens_seen": 21626880 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032837301587301587, | |
| "loss": 5.0504, | |
| "theoretical_loss": 6.034316715356304, | |
| "tokens_seen": 21692416 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00032936507936507937, | |
| "loss": 5.0898, | |
| "theoretical_loss": 6.031255809922974, | |
| "tokens_seen": 21757952 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00033035714285714287, | |
| "loss": 5.0121, | |
| "theoretical_loss": 6.0282066828801195, | |
| "tokens_seen": 21823488 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003313492063492063, | |
| "loss": 5.1363, | |
| "theoretical_loss": 6.0251692537374595, | |
| "tokens_seen": 21889024 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003323412698412698, | |
| "loss": 5.1503, | |
| "theoretical_loss": 6.022143442793643, | |
| "tokens_seen": 21954560 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003333333333333333, | |
| "loss": 5.1066, | |
| "theoretical_loss": 6.0191291711261945, | |
| "tokens_seen": 22020096 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003343253968253968, | |
| "loss": 5.0889, | |
| "theoretical_loss": 6.016126360581609, | |
| "tokens_seen": 22085632 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003353174603174603, | |
| "loss": 4.9077, | |
| "theoretical_loss": 6.013134933765605, | |
| "tokens_seen": 22151168 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003363095238095238, | |
| "loss": 5.2813, | |
| "theoretical_loss": 6.010154814033528, | |
| "tokens_seen": 22216704 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00033730158730158733, | |
| "loss": 4.6999, | |
| "theoretical_loss": 6.007185925480902, | |
| "tokens_seen": 22282240 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00033829365079365083, | |
| "loss": 5.0728, | |
| "theoretical_loss": 6.0042281929341375, | |
| "tokens_seen": 22347776 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00033928571428571433, | |
| "loss": 5.1319, | |
| "theoretical_loss": 6.001281541941363, | |
| "tokens_seen": 22413312 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003402777777777778, | |
| "loss": 5.0915, | |
| "theoretical_loss": 5.998345898763421, | |
| "tokens_seen": 22478848 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003412698412698413, | |
| "loss": 4.8912, | |
| "theoretical_loss": 5.995421190364983, | |
| "tokens_seen": 22544384 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003422619047619048, | |
| "loss": 5.0317, | |
| "theoretical_loss": 5.992507344405814, | |
| "tokens_seen": 22609920 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003432539682539683, | |
| "loss": 5.1603, | |
| "theoretical_loss": 5.9896042892321635, | |
| "tokens_seen": 22675456 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00034424603174603173, | |
| "loss": 4.9822, | |
| "theoretical_loss": 5.986711953868287, | |
| "tokens_seen": 22740992 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00034523809523809523, | |
| "loss": 5.3948, | |
| "theoretical_loss": 5.983830268008107, | |
| "tokens_seen": 22806528 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00034623015873015873, | |
| "loss": 5.222, | |
| "theoretical_loss": 5.980959162006979, | |
| "tokens_seen": 22872064 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 24859, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.439995288848877, | |
| "objective/train/theoretical_loss": 5.978098566873603, | |
| "objective/train/tokens_used": 43397600, | |
| "theoretical_loss": 5.978098566873603, | |
| "tokens_seen": 22937600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00034722222222222224, | |
| "loss": 4.9666, | |
| "theoretical_loss": 5.978098566873603, | |
| "tokens_seen": 22937600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003482142857142857, | |
| "loss": 4.7873, | |
| "theoretical_loss": 5.975248414262053, | |
| "tokens_seen": 23003136 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003492063492063492, | |
| "loss": 5.1429, | |
| "theoretical_loss": 5.972408636463909, | |
| "tokens_seen": 23068672 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003501984126984127, | |
| "loss": 4.8387, | |
| "theoretical_loss": 5.969579166400528, | |
| "tokens_seen": 23134208 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003511904761904762, | |
| "loss": 5.2741, | |
| "theoretical_loss": 5.966759937615427, | |
| "tokens_seen": 23199744 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003521825396825397, | |
| "loss": 4.9993, | |
| "theoretical_loss": 5.96395088426676, | |
| "tokens_seen": 23265280 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003531746031746032, | |
| "loss": 4.838, | |
| "theoretical_loss": 5.961151941119932, | |
| "tokens_seen": 23330816 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003541666666666667, | |
| "loss": 4.8937, | |
| "theoretical_loss": 5.95836304354031, | |
| "tokens_seen": 23396352 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003551587301587302, | |
| "loss": 5.1755, | |
| "theoretical_loss": 5.9555841274860395, | |
| "tokens_seen": 23461888 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003561507936507937, | |
| "loss": 5.0192, | |
| "theoretical_loss": 5.952815129500973, | |
| "tokens_seen": 23527424 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035714285714285714, | |
| "loss": 5.1528, | |
| "theoretical_loss": 5.950055986707699, | |
| "tokens_seen": 23592960 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035813492063492064, | |
| "loss": 5.2473, | |
| "theoretical_loss": 5.9473066368006755, | |
| "tokens_seen": 23658496 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035912698412698415, | |
| "loss": 5.1431, | |
| "theoretical_loss": 5.944567018039454, | |
| "tokens_seen": 23724032 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00036011904761904765, | |
| "loss": 5.2078, | |
| "theoretical_loss": 5.941837069242023, | |
| "tokens_seen": 23789568 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003611111111111111, | |
| "loss": 5.2074, | |
| "theoretical_loss": 5.939116729778224, | |
| "tokens_seen": 23855104 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003621031746031746, | |
| "loss": 5.045, | |
| "theoretical_loss": 5.936405939563279, | |
| "tokens_seen": 23920640 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003630952380952381, | |
| "loss": 5.0272, | |
| "theoretical_loss": 5.93370463905141, | |
| "tokens_seen": 23986176 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003640873015873016, | |
| "loss": 5.0628, | |
| "theoretical_loss": 5.931012769229536, | |
| "tokens_seen": 24051712 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00036507936507936505, | |
| "loss": 4.6127, | |
| "theoretical_loss": 5.928330271611081, | |
| "tokens_seen": 24117248 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00036607142857142855, | |
| "loss": 4.8408, | |
| "theoretical_loss": 5.925657088229862, | |
| "tokens_seen": 24182784 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00036706349206349205, | |
| "loss": 4.9603, | |
| "theoretical_loss": 5.9229931616340545, | |
| "tokens_seen": 24248320 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003680555555555556, | |
| "loss": 5.0127, | |
| "theoretical_loss": 5.920338434880263, | |
| "tokens_seen": 24313856 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00036904761904761905, | |
| "loss": 5.3699, | |
| "theoretical_loss": 5.9176928515276535, | |
| "tokens_seen": 24379392 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037003968253968255, | |
| "loss": 4.9974, | |
| "theoretical_loss": 5.915056355632197, | |
| "tokens_seen": 24444928 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037103174603174606, | |
| "loss": 4.9348, | |
| "theoretical_loss": 5.912428891740967, | |
| "tokens_seen": 24510464 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 25411, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.414747714996338, | |
| "objective/train/theoretical_loss": 5.90981040488653, | |
| "objective/train/tokens_used": 45036000, | |
| "theoretical_loss": 5.90981040488653, | |
| "tokens_seen": 24576000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037202380952380956, | |
| "loss": 5.0376, | |
| "theoretical_loss": 5.90981040488653, | |
| "tokens_seen": 24576000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000373015873015873, | |
| "loss": 5.0724, | |
| "theoretical_loss": 5.907200840581417, | |
| "tokens_seen": 24641536 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003740079365079365, | |
| "loss": 5.2247, | |
| "theoretical_loss": 5.904600144812672, | |
| "tokens_seen": 24707072 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000375, | |
| "loss": 5.4811, | |
| "theoretical_loss": 5.902008264036468, | |
| "tokens_seen": 24772608 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003759920634920635, | |
| "loss": 4.9107, | |
| "theoretical_loss": 5.899425145172803, | |
| "tokens_seen": 24838144 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000376984126984127, | |
| "loss": 5.0933, | |
| "theoretical_loss": 5.896850735600281, | |
| "tokens_seen": 24903680 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037797619047619046, | |
| "loss": 5.1802, | |
| "theoretical_loss": 5.8942849831509445, | |
| "tokens_seen": 24969216 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037896825396825396, | |
| "loss": 5.2521, | |
| "theoretical_loss": 5.891727836105194, | |
| "tokens_seen": 25034752 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037996031746031746, | |
| "loss": 4.8607, | |
| "theoretical_loss": 5.889179243186776, | |
| "tokens_seen": 25100288 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00038095238095238096, | |
| "loss": 5.0973, | |
| "theoretical_loss": 5.886639153557828, | |
| "tokens_seen": 25165824 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003819444444444444, | |
| "loss": 5.3397, | |
| "theoretical_loss": 5.8841075168140105, | |
| "tokens_seen": 25231360 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00038293650793650797, | |
| "loss": 4.7777, | |
| "theoretical_loss": 5.88158428297969, | |
| "tokens_seen": 25296896 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00038392857142857147, | |
| "loss": 4.7219, | |
| "theoretical_loss": 5.879069402503189, | |
| "tokens_seen": 25362432 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00038492063492063497, | |
| "loss": 5.0299, | |
| "theoretical_loss": 5.876562826252119, | |
| "tokens_seen": 25427968 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003859126984126984, | |
| "loss": 4.8369, | |
| "theoretical_loss": 5.874064505508748, | |
| "tokens_seen": 25493504 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003869047619047619, | |
| "loss": 5.0017, | |
| "theoretical_loss": 5.871574391965453, | |
| "tokens_seen": 25559040 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003878968253968254, | |
| "loss": 4.9705, | |
| "theoretical_loss": 5.8690924377202265, | |
| "tokens_seen": 25624576 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003888888888888889, | |
| "loss": 5.034, | |
| "theoretical_loss": 5.866618595272241, | |
| "tokens_seen": 25690112 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00038988095238095237, | |
| "loss": 4.7181, | |
| "theoretical_loss": 5.864152817517482, | |
| "tokens_seen": 25755648 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00039087301587301587, | |
| "loss": 4.9562, | |
| "theoretical_loss": 5.8616950577444245, | |
| "tokens_seen": 25821184 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00039186507936507937, | |
| "loss": 4.9405, | |
| "theoretical_loss": 5.859245269629783, | |
| "tokens_seen": 25886720 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003928571428571429, | |
| "loss": 5.099, | |
| "theoretical_loss": 5.85680340723431, | |
| "tokens_seen": 25952256 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003938492063492063, | |
| "loss": 4.6503, | |
| "theoretical_loss": 5.85436942499865, | |
| "tokens_seen": 26017792 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003948412698412698, | |
| "loss": 4.8672, | |
| "theoretical_loss": 5.851943277739253, | |
| "tokens_seen": 26083328 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003958333333333333, | |
| "loss": 5.0201, | |
| "theoretical_loss": 5.84952492064434, | |
| "tokens_seen": 26148864 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 26668, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 5.337137222290039, | |
| "objective/train/theoretical_loss": 5.847114309269919, | |
| "objective/train/tokens_used": 46674400, | |
| "theoretical_loss": 5.847114309269919, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003968253968253968, | |
| "loss": 5.1138, | |
| "theoretical_loss": 5.847114309269919, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003978174603174603, | |
| "loss": 4.8239, | |
| "theoretical_loss": 5.844711399535855, | |
| "tokens_seen": 26279936 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00039880952380952383, | |
| "loss": 4.7304, | |
| "theoretical_loss": 5.842316147722, | |
| "tokens_seen": 26345472 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00039980158730158733, | |
| "loss": 5.0092, | |
| "theoretical_loss": 5.839928510464356, | |
| "tokens_seen": 26411008 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040079365079365083, | |
| "loss": 4.4343, | |
| "theoretical_loss": 5.837548444751306, | |
| "tokens_seen": 26476544 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040178571428571433, | |
| "loss": 4.9309, | |
| "theoretical_loss": 5.835175907919885, | |
| "tokens_seen": 26542080 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004027777777777778, | |
| "loss": 5.0148, | |
| "theoretical_loss": 5.832810857652097, | |
| "tokens_seen": 26607616 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004037698412698413, | |
| "loss": 4.9709, | |
| "theoretical_loss": 5.830453251971296, | |
| "tokens_seen": 26673152 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004047619047619048, | |
| "loss": 5.1225, | |
| "theoretical_loss": 5.8281030492385835, | |
| "tokens_seen": 26738688 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004057539682539683, | |
| "loss": 4.9439, | |
| "theoretical_loss": 5.82576020814929, | |
| "tokens_seen": 26804224 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040674603174603173, | |
| "loss": 5.154, | |
| "theoretical_loss": 5.823424687729469, | |
| "tokens_seen": 26869760 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040773809523809523, | |
| "loss": 4.6484, | |
| "theoretical_loss": 5.821096447332456, | |
| "tokens_seen": 26935296 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040873015873015874, | |
| "loss": 4.6262, | |
| "theoretical_loss": 5.818775446635469, | |
| "tokens_seen": 27000832 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00040972222222222224, | |
| "loss": 5.0236, | |
| "theoretical_loss": 5.816461645636238, | |
| "tokens_seen": 27066368 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004107142857142857, | |
| "loss": 5.1234, | |
| "theoretical_loss": 5.814155004649702, | |
| "tokens_seen": 27131904 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004117063492063492, | |
| "loss": 4.6509, | |
| "theoretical_loss": 5.811855484304724, | |
| "tokens_seen": 27197440 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004126984126984127, | |
| "loss": 4.9814, | |
| "theoretical_loss": 5.809563045540864, | |
| "tokens_seen": 27262976 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004136904761904762, | |
| "loss": 5.0187, | |
| "theoretical_loss": 5.8072776496051866, | |
| "tokens_seen": 27328512 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004146825396825397, | |
| "loss": 4.9255, | |
| "theoretical_loss": 5.804999258049106, | |
| "tokens_seen": 27394048 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004156746031746032, | |
| "loss": 5.0265, | |
| "theoretical_loss": 5.802727832725283, | |
| "tokens_seen": 27459584 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004166666666666667, | |
| "loss": 4.8827, | |
| "theoretical_loss": 5.800463335784541, | |
| "tokens_seen": 27525120 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004176587301587302, | |
| "loss": 5.0962, | |
| "theoretical_loss": 5.798205729672842, | |
| "tokens_seen": 27590656 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004186507936507937, | |
| "loss": 5.0409, | |
| "theoretical_loss": 5.795954977128286, | |
| "tokens_seen": 27656192 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00041964285714285714, | |
| "loss": 4.8332, | |
| "theoretical_loss": 5.7937110411781525, | |
| "tokens_seen": 27721728 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042063492063492065, | |
| "loss": 4.813, | |
| "theoretical_loss": 5.79147388513598, | |
| "tokens_seen": 27787264 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 27295, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.60736083984375, | |
| "objective/train/theoretical_loss": 5.789243472598683, | |
| "objective/train/tokens_used": 48312800, | |
| "theoretical_loss": 5.789243472598683, | |
| "tokens_seen": 27852800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042162698412698415, | |
| "loss": 4.735, | |
| "theoretical_loss": 5.789243472598683, | |
| "tokens_seen": 27852800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042261904761904765, | |
| "loss": 5.0103, | |
| "theoretical_loss": 5.787019767443696, | |
| "tokens_seen": 27918336 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004236111111111111, | |
| "loss": 4.9487, | |
| "theoretical_loss": 5.784802733826166, | |
| "tokens_seen": 27983872 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004246031746031746, | |
| "loss": 5.0393, | |
| "theoretical_loss": 5.782592336176171, | |
| "tokens_seen": 28049408 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004255952380952381, | |
| "loss": 4.8239, | |
| "theoretical_loss": 5.780388539195972, | |
| "tokens_seen": 28114944 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004265873015873016, | |
| "loss": 4.8301, | |
| "theoretical_loss": 5.778191307857307, | |
| "tokens_seen": 28180480 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042757936507936505, | |
| "loss": 4.8818, | |
| "theoretical_loss": 5.776000607398707, | |
| "tokens_seen": 28246016 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042857142857142855, | |
| "loss": 5.0365, | |
| "theoretical_loss": 5.773816403322854, | |
| "tokens_seen": 28311552 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00042956349206349205, | |
| "loss": 4.8806, | |
| "theoretical_loss": 5.7716386613939665, | |
| "tokens_seen": 28377088 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004305555555555556, | |
| "loss": 4.8212, | |
| "theoretical_loss": 5.769467347635221, | |
| "tokens_seen": 28442624 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00043154761904761905, | |
| "loss": 4.6856, | |
| "theoretical_loss": 5.767302428326196, | |
| "tokens_seen": 28508160 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00043253968253968256, | |
| "loss": 5.1099, | |
| "theoretical_loss": 5.765143870000358, | |
| "tokens_seen": 28573696 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00043353174603174606, | |
| "loss": 4.9942, | |
| "theoretical_loss": 5.762991639442574, | |
| "tokens_seen": 28639232 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00043452380952380956, | |
| "loss": 4.7269, | |
| "theoretical_loss": 5.7608457036866465, | |
| "tokens_seen": 28704768 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000435515873015873, | |
| "loss": 4.7079, | |
| "theoretical_loss": 5.758706030012889, | |
| "tokens_seen": 28770304 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004365079365079365, | |
| "loss": 5.0535, | |
| "theoretical_loss": 5.756572585945728, | |
| "tokens_seen": 28835840 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004375, | |
| "loss": 4.9953, | |
| "theoretical_loss": 5.754445339251326, | |
| "tokens_seen": 28901376 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004384920634920635, | |
| "loss": 4.8239, | |
| "theoretical_loss": 5.752324257935244, | |
| "tokens_seen": 28966912 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000439484126984127, | |
| "loss": 4.8738, | |
| "theoretical_loss": 5.750209310240125, | |
| "tokens_seen": 29032448 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044047619047619046, | |
| "loss": 4.5774, | |
| "theoretical_loss": 5.7481004646434055, | |
| "tokens_seen": 29097984 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044146825396825396, | |
| "loss": 4.7821, | |
| "theoretical_loss": 5.745997689855058, | |
| "tokens_seen": 29163520 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044246031746031746, | |
| "loss": 4.9197, | |
| "theoretical_loss": 5.743900954815356, | |
| "tokens_seen": 29229056 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044345238095238096, | |
| "loss": 4.9178, | |
| "theoretical_loss": 5.741810228692663, | |
| "tokens_seen": 29294592 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004444444444444444, | |
| "loss": 4.9811, | |
| "theoretical_loss": 5.739725480881262, | |
| "tokens_seen": 29360128 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044543650793650797, | |
| "loss": 4.595, | |
| "theoretical_loss": 5.737646680999193, | |
| "tokens_seen": 29425664 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 28553, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.193212032318115, | |
| "objective/train/theoretical_loss": 5.73557379888612, | |
| "objective/train/tokens_used": 49951200, | |
| "theoretical_loss": 5.73557379888612, | |
| "tokens_seen": 29491200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044642857142857147, | |
| "loss": 4.6289, | |
| "theoretical_loss": 5.73557379888612, | |
| "tokens_seen": 29491200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00044742063492063497, | |
| "loss": 4.5175, | |
| "theoretical_loss": 5.733506804601236, | |
| "tokens_seen": 29556736 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004484126984126984, | |
| "loss": 4.97, | |
| "theoretical_loss": 5.7314456684211725, | |
| "tokens_seen": 29622272 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004494047619047619, | |
| "loss": 4.9007, | |
| "theoretical_loss": 5.729390360837952, | |
| "tokens_seen": 29687808 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004503968253968254, | |
| "loss": 5.0718, | |
| "theoretical_loss": 5.7273408525569485, | |
| "tokens_seen": 29753344 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004513888888888889, | |
| "loss": 4.8076, | |
| "theoretical_loss": 5.725297114494884, | |
| "tokens_seen": 29818880 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00045238095238095237, | |
| "loss": 4.9151, | |
| "theoretical_loss": 5.7232591177778405, | |
| "tokens_seen": 29884416 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00045337301587301587, | |
| "loss": 4.7611, | |
| "theoretical_loss": 5.7212268337393, | |
| "tokens_seen": 29949952 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00045436507936507937, | |
| "loss": 4.6533, | |
| "theoretical_loss": 5.719200233918203, | |
| "tokens_seen": 30015488 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004553571428571429, | |
| "loss": 4.7689, | |
| "theoretical_loss": 5.717179290057032, | |
| "tokens_seen": 30081024 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004563492063492063, | |
| "loss": 4.753, | |
| "theoretical_loss": 5.715163974099917, | |
| "tokens_seen": 30146560 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004573412698412698, | |
| "loss": 4.9783, | |
| "theoretical_loss": 5.713154258190757, | |
| "tokens_seen": 30212096 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004583333333333333, | |
| "loss": 4.8919, | |
| "theoretical_loss": 5.711150114671375, | |
| "tokens_seen": 30277632 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004593253968253968, | |
| "loss": 4.6246, | |
| "theoretical_loss": 5.709151516079683, | |
| "tokens_seen": 30343168 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046031746031746033, | |
| "loss": 4.8047, | |
| "theoretical_loss": 5.707158435147875, | |
| "tokens_seen": 30408704 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046130952380952383, | |
| "loss": 4.7619, | |
| "theoretical_loss": 5.705170844800628, | |
| "tokens_seen": 30474240 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046230158730158733, | |
| "loss": 5.0389, | |
| "theoretical_loss": 5.703188718153347, | |
| "tokens_seen": 30539776 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046329365079365083, | |
| "loss": 4.7978, | |
| "theoretical_loss": 5.7012120285104055, | |
| "tokens_seen": 30605312 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046428571428571433, | |
| "loss": 4.8182, | |
| "theoretical_loss": 5.699240749363417, | |
| "tokens_seen": 30670848 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004652777777777778, | |
| "loss": 5.1046, | |
| "theoretical_loss": 5.69727485438953, | |
| "tokens_seen": 30736384 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004662698412698413, | |
| "loss": 4.9301, | |
| "theoretical_loss": 5.695314317449732, | |
| "tokens_seen": 30801920 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004672619047619048, | |
| "loss": 4.9513, | |
| "theoretical_loss": 5.69335911258718, | |
| "tokens_seen": 30867456 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004682539682539683, | |
| "loss": 4.7506, | |
| "theoretical_loss": 5.691409214025544, | |
| "tokens_seen": 30932992 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046924603174603173, | |
| "loss": 4.9203, | |
| "theoretical_loss": 5.689464596167383, | |
| "tokens_seen": 30998528 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00047023809523809523, | |
| "loss": 4.6588, | |
| "theoretical_loss": 5.687525233592513, | |
| "tokens_seen": 31064064 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 29210, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.980744361877441, | |
| "objective/train/theoretical_loss": 5.685591101056422, | |
| "objective/train/tokens_used": 51589600, | |
| "theoretical_loss": 5.685591101056422, | |
| "tokens_seen": 31129600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00047123015873015874, | |
| "loss": 4.8131, | |
| "theoretical_loss": 5.685591101056422, | |
| "tokens_seen": 31129600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00047222222222222224, | |
| "loss": 4.9729, | |
| "theoretical_loss": 5.683662173488678, | |
| "tokens_seen": 31195136 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004732142857142857, | |
| "loss": 4.9988, | |
| "theoretical_loss": 5.681738425991377, | |
| "tokens_seen": 31260672 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004742063492063492, | |
| "loss": 4.9296, | |
| "theoretical_loss": 5.679819833837586, | |
| "tokens_seen": 31326208 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004751984126984127, | |
| "loss": 4.8866, | |
| "theoretical_loss": 5.677906372469826, | |
| "tokens_seen": 31391744 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004761904761904762, | |
| "loss": 4.8619, | |
| "theoretical_loss": 5.675998017498549, | |
| "tokens_seen": 31457280 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004771825396825397, | |
| "loss": 4.4881, | |
| "theoretical_loss": 5.674094744700648, | |
| "tokens_seen": 31522816 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004781746031746032, | |
| "loss": 4.7385, | |
| "theoretical_loss": 5.672196530017979, | |
| "tokens_seen": 31588352 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004791666666666667, | |
| "loss": 4.9446, | |
| "theoretical_loss": 5.670303349555893, | |
| "tokens_seen": 31653888 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004801587301587302, | |
| "loss": 4.5805, | |
| "theoretical_loss": 5.668415179581795, | |
| "tokens_seen": 31719424 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004811507936507937, | |
| "loss": 4.943, | |
| "theoretical_loss": 5.666531996523711, | |
| "tokens_seen": 31784960 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00048214285714285715, | |
| "loss": 4.9994, | |
| "theoretical_loss": 5.664653776968867, | |
| "tokens_seen": 31850496 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00048313492063492065, | |
| "loss": 4.9432, | |
| "theoretical_loss": 5.6627804976622995, | |
| "tokens_seen": 31916032 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00048412698412698415, | |
| "loss": 4.4612, | |
| "theoretical_loss": 5.660912135505461, | |
| "tokens_seen": 31981568 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00048511904761904765, | |
| "loss": 5.0296, | |
| "theoretical_loss": 5.659048667554853, | |
| "tokens_seen": 32047104 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004861111111111111, | |
| "loss": 4.7342, | |
| "theoretical_loss": 5.657190071020672, | |
| "tokens_seen": 32112640 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004871031746031746, | |
| "loss": 5.0271, | |
| "theoretical_loss": 5.65533632326547, | |
| "tokens_seen": 32178176 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004880952380952381, | |
| "loss": 4.5208, | |
| "theoretical_loss": 5.6534874018028205, | |
| "tokens_seen": 32243712 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004890873015873016, | |
| "loss": 4.8883, | |
| "theoretical_loss": 5.651643284296018, | |
| "tokens_seen": 32309248 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004900793650793651, | |
| "loss": 4.5918, | |
| "theoretical_loss": 5.6498039485567695, | |
| "tokens_seen": 32374784 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004910714285714286, | |
| "loss": 4.8362, | |
| "theoretical_loss": 5.6479693725439155, | |
| "tokens_seen": 32440320 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000492063492063492, | |
| "loss": 4.4816, | |
| "theoretical_loss": 5.646139534362161, | |
| "tokens_seen": 32505856 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004930555555555556, | |
| "loss": 4.7304, | |
| "theoretical_loss": 5.644314412260815, | |
| "tokens_seen": 32571392 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004940476190476191, | |
| "loss": 4.7639, | |
| "theoretical_loss": 5.642493984632544, | |
| "tokens_seen": 32636928 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004950396825396826, | |
| "loss": 4.486, | |
| "theoretical_loss": 5.640678230012151, | |
| "tokens_seen": 32702464 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "objective/train/docs_used": 30192, | |
| "objective/train/instantaneous_batch_size": 16, | |
| "objective/train/instantaneous_microbatch_size": 16384, | |
| "objective/train/original_loss": 4.205135822296143, | |
| "objective/train/theoretical_loss": 5.638867127075349, | |
| "objective/train/tokens_used": 53228000, | |
| "theoretical_loss": 5.638867127075349, | |
| "tokens_seen": 32768000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000496031746031746, | |
| "loss": 4.6383, | |
| "theoretical_loss": 5.638867127075349, | |
| "tokens_seen": 32768000 | |
| } | |
| ], | |
| "max_steps": 50354, | |
| "num_train_epochs": 9223372036854775807, | |
| "total_flos": 1.6722690048e+16, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |