diff --git "a/checkpoint-5000/trainer_state.json" "b/checkpoint-5000/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-5000/trainer_state.json" @@ -0,0 +1,41024 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.19859395480001588, + "global_step": 5000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-06, + "loss": 4.3952, + "theoretical_loss": 17.566201104328645, + "tokens_seen": 131072 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-06, + "loss": 4.3745, + "theoretical_loss": 14.89231675598857, + "tokens_seen": 262144 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809525e-06, + "loss": 4.1183, + "theoretical_loss": 13.552561472550224, + "tokens_seen": 393216 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 3.9369, + "theoretical_loss": 12.690129625483323, + "tokens_seen": 524288 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-06, + "loss": 4.0965, + "theoretical_loss": 12.067412607035077, + "tokens_seen": 655360 + }, + { + "epoch": 0.0, + "learning_rate": 1.1904761904761905e-05, + "loss": 3.5987, + "theoretical_loss": 11.586719208706729, + "tokens_seen": 786432 + }, + { + "epoch": 0.0, + "learning_rate": 1.3888888888888888e-05, + "loss": 3.6811, + "theoretical_loss": 11.199011702111871, + "tokens_seen": 917504 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 3.7779, + "theoretical_loss": 10.87642808645695, + "tokens_seen": 1048576 + }, + { + "epoch": 0.0, + "learning_rate": 1.7857142857142855e-05, + "loss": 3.8919, + "theoretical_loss": 10.60172987623028, + "tokens_seen": 1179648 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-05, + "loss": 4.0009, + "theoretical_loss": 10.36356394376333, + "tokens_seen": 1310720 + }, + { + "epoch": 0.0, + "learning_rate": 2.1825396825396824e-05, + "loss": 3.75, + "theoretical_loss": 10.15408655327002, + "tokens_seen": 1441792 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 3.8353, + "theoretical_loss": 9.967669178840278, + "tokens_seen": 1572864 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.36052054166793823, + "objective/train/avg_token_score": 0.01564830169081688, + "objective/train/docs_used": 13112, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 3.4145798683166504, + "objective/train/original_loss": 9.366209983825684, + "objective/train/theoretical_loss": 9.881763126393109, + "objective/train/tokens_used": 1638400, + "objective/train/unlikelihood_loss": 0.004730077460408211, + "theoretical_loss": 9.881763126393109, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 2.5793650793650793e-05, + "loss": 3.9383, + "theoretical_loss": 9.80014659154056, + "tokens_seen": 1703936 + }, + { + "epoch": 0.0, + "learning_rate": 2.7777777777777776e-05, + "loss": 3.6506, + "theoretical_loss": 9.648356759081546, + "tokens_seen": 1835008 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904762e-05, + "loss": 3.5887, + "theoretical_loss": 9.509847046764852, + "tokens_seen": 1966080 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 3.7839, + "theoretical_loss": 9.382679790910457, + "tokens_seen": 2097152 + }, + { + "epoch": 0.0, + "learning_rate": 3.373015873015873e-05, + "loss": 3.3506, + "theoretical_loss": 9.265299666660276, + "tokens_seen": 2228224 + }, + { + "epoch": 0.0, + "learning_rate": 3.571428571428571e-05, + "loss": 3.6369, + "theoretical_loss": 9.156440812508292, + "tokens_seen": 2359296 + }, + { + "epoch": 0.0, + "learning_rate": 3.76984126984127e-05, + "loss": 3.1405, + "theoretical_loss": 9.055060296533734, + "tokens_seen": 2490368 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 3.5961, + "theoretical_loss": 8.960289489909357, + "tokens_seen": 2621440 + }, + { + "epoch": 0.0, + "learning_rate": 4.1666666666666665e-05, + "loss": 3.8762, + "theoretical_loss": 8.871397892478225, + "tokens_seen": 2752512 + }, + { + "epoch": 0.0, + "learning_rate": 4.365079365079365e-05, + "loss": 3.1495, + "theoretical_loss": 8.787765792778412, + "tokens_seen": 2883584 + }, + { + "epoch": 0.0, + "learning_rate": 4.563492063492063e-05, + "loss": 3.2901, + "theoretical_loss": 8.708863309520833, + "tokens_seen": 3014656 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 3.7481, + "theoretical_loss": 8.634234117735474, + "tokens_seen": 3145728 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.3303396999835968, + "objective/train/avg_token_score": 0.04211259260773659, + "objective/train/docs_used": 13755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.995987892150879, + "objective/train/original_loss": 9.143326759338379, + "objective/train/theoretical_loss": 8.563482664611069, + "objective/train/tokens_used": 3276800, + "objective/train/unlikelihood_loss": 0.01486299093812704, + "theoretical_loss": 8.563482664611069, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-05, + "loss": 3.5744, + "theoretical_loss": 8.563482664611069, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 5.1587301587301586e-05, + "loss": 3.5767, + "theoretical_loss": 8.496264019646002, + "tokens_seen": 3407872 + }, + { + "epoch": 0.0, + "learning_rate": 5.357142857142857e-05, + "loss": 3.0622, + "theoretical_loss": 8.432275737672779, + "tokens_seen": 3538944 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555555e-05, + "loss": 3.7336, + "theoretical_loss": 8.371251277120209, + "tokens_seen": 3670016 + }, + { + "epoch": 0.0, + "learning_rate": 5.753968253968254e-05, + "loss": 3.411, + "theoretical_loss": 8.31295463228533, + "tokens_seen": 3801088 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809524e-05, + "loss": 3.38, + "theoretical_loss": 8.257175922251864, + "tokens_seen": 3932160 + }, + { + "epoch": 0.0, + "learning_rate": 6.15079365079365e-05, + "loss": 3.1689, + "theoretical_loss": 8.20372774027797, + "tokens_seen": 4063232 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 3.3772, + "theoretical_loss": 8.152442112639616, + "tokens_seen": 4194304 + }, + { + "epoch": 0.0, + "learning_rate": 6.547619047619048e-05, + "loss": 2.919, + "theoretical_loss": 8.10316794961571, + "tokens_seen": 4325376 + }, + { + "epoch": 0.0, + "learning_rate": 6.746031746031745e-05, + "loss": 3.1994, + "theoretical_loss": 8.055768896701416, + "tokens_seen": 4456448 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-05, + "loss": 3.6138, + "theoretical_loss": 8.010121513461836, + "tokens_seen": 4587520 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857142e-05, + "loss": 3.0735, + "theoretical_loss": 7.966113722271801, + "tokens_seen": 4718592 + }, + { + "epoch": 0.0, + "learning_rate": 7.341269841269842e-05, + "loss": 3.063, + "theoretical_loss": 7.9236434806675184, + "tokens_seen": 4849664 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.3146994113922119, + "objective/train/avg_token_score": 0.016941465437412262, + "objective/train/docs_used": 15099, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.699511766433716, + "objective/train/original_loss": 8.369515419006348, + "objective/train/theoretical_loss": 7.902955512484067, + "objective/train/tokens_used": 4915200, + "objective/train/unlikelihood_loss": 0.032859548926353455, + "theoretical_loss": 7.902955512484067, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 7.53968253968254e-05, + "loss": 3.196, + "theoretical_loss": 7.882617639989203, + "tokens_seen": 4980736 + }, + { + "epoch": 0.0, + "learning_rate": 7.738095238095239e-05, + "loss": 2.9935, + "theoretical_loss": 7.842950960027937, + "tokens_seen": 5111808 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 3.4802, + "theoretical_loss": 7.804565254954165, + "tokens_seen": 5242880 + }, + { + "epoch": 0.0, + "learning_rate": 8.134920634920635e-05, + "loss": 3.2085, + "theoretical_loss": 7.767388650235364, + "tokens_seen": 5373952 + }, + { + "epoch": 0.0, + "learning_rate": 8.333333333333333e-05, + "loss": 3.3755, + "theoretical_loss": 7.731354933799318, + "tokens_seen": 5505024 + }, + { + "epoch": 0.0, + "learning_rate": 8.531746031746032e-05, + "loss": 3.6461, + "theoretical_loss": 7.696402987558934, + "tokens_seen": 5636096 + }, + { + "epoch": 0.0, + "learning_rate": 8.73015873015873e-05, + "loss": 3.0517, + "theoretical_loss": 7.662476287731328, + "tokens_seen": 5767168 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571429e-05, + "loss": 3.0863, + "theoretical_loss": 7.629522464270861, + "tokens_seen": 5898240 + }, + { + "epoch": 0.0, + "learning_rate": 9.126984126984126e-05, + "loss": 2.7058, + "theoretical_loss": 7.59749291128028, + "tokens_seen": 6029312 + }, + { + "epoch": 0.0, + "learning_rate": 9.325396825396825e-05, + "loss": 3.3358, + "theoretical_loss": 7.5663424415343705, + "tokens_seen": 6160384 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 3.0732, + "theoretical_loss": 7.536028979299919, + "tokens_seen": 6291456 + }, + { + "epoch": 0.0, + "learning_rate": 9.722222222222223e-05, + "loss": 3.0706, + "theoretical_loss": 7.506513286506497, + "tokens_seen": 6422528 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.41287878155708313, + "objective/train/avg_token_score": 0.03952885419130325, + "objective/train/docs_used": 15777, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 3.0034613609313965, + "objective/train/original_loss": 7.385845184326172, + "objective/train/theoretical_loss": 7.4777587180480305, + "objective/train/tokens_used": 6553600, + "objective/train/unlikelihood_loss": 0.09759358316659927, + "theoretical_loss": 7.4777587180480305, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-05, + "loss": 3.0284, + "theoretical_loss": 7.4777587180480305, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001011904761904762, + "loss": 2.8636, + "theoretical_loss": 7.449731002601916, + "tokens_seen": 6684672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460317, + "loss": 2.724, + "theoretical_loss": 7.422398045861905, + "tokens_seen": 6815744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010515873015873016, + "loss": 2.9955, + "theoretical_loss": 7.395729753510345, + "tokens_seen": 6946816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010714285714285714, + "loss": 2.967, + "theoretical_loss": 7.369697871618373, + "tokens_seen": 7077888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010912698412698413, + "loss": 3.2671, + "theoretical_loss": 7.3442758424706875, + "tokens_seen": 7208960 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001111111111111111, + "loss": 3.0036, + "theoretical_loss": 7.319438674073677, + "tokens_seen": 7340032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011309523809523809, + "loss": 3.0592, + "theoretical_loss": 7.295162821829564, + "tokens_seen": 7471104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011507936507936508, + "loss": 3.2368, + "theoretical_loss": 7.271426081050832, + "tokens_seen": 7602176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011706349206349207, + "loss": 3.1647, + "theoretical_loss": 7.24820748915387, + "tokens_seen": 7733248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 2.9356, + "theoretical_loss": 7.225487236512497, + "tokens_seen": 7864320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012103174603174604, + "loss": 2.8582, + "theoretical_loss": 7.2032465850744005, + "tokens_seen": 7995392 + }, + { + "epoch": 0.0, + "learning_rate": 0.000123015873015873, + "loss": 2.7811, + "theoretical_loss": 7.1814677939495155, + "tokens_seen": 8126464 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.4550953209400177, + "objective/train/avg_token_score": 0.02127336896955967, + "objective/train/docs_used": 17166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 3.3246047496795654, + "objective/train/original_loss": 7.3253278732299805, + "objective/train/theoretical_loss": 7.1707463077509646, + "objective/train/tokens_used": 8192000, + "objective/train/unlikelihood_loss": 0.044195178896188736, + "theoretical_loss": 7.1707463077509646, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000125, + "loss": 3.3126, + "theoretical_loss": 7.160134051271272, + "tokens_seen": 8257536 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 3.0628, + "theoretical_loss": 7.139229411711638, + "tokens_seen": 8388608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012896825396825398, + "loss": 2.7911, + "theoretical_loss": 7.118738739100616, + "tokens_seen": 8519680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013095238095238096, + "loss": 2.8125, + "theoretical_loss": 7.09864765366177, + "tokens_seen": 8650752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013293650793650793, + "loss": 2.6533, + "theoretical_loss": 7.078942483428749, + "tokens_seen": 8781824 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001349206349206349, + "loss": 2.9263, + "theoretical_loss": 7.059610219454568, + "tokens_seen": 8912896 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001369047619047619, + "loss": 2.8425, + "theoretical_loss": 7.040638474466625, + "tokens_seen": 9043968 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001388888888888889, + "loss": 3.1806, + "theoretical_loss": 7.022015444656678, + "tokens_seen": 9175040 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014087301587301586, + "loss": 3.0648, + "theoretical_loss": 7.003729874327071, + "tokens_seen": 9306112 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714284, + "loss": 3.0663, + "theoretical_loss": 6.98577102314278, + "tokens_seen": 9437184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014484126984126984, + "loss": 2.7552, + "theoretical_loss": 6.968128635764015, + "tokens_seen": 9568256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014682539682539685, + "loss": 3.0583, + "theoretical_loss": 6.950792913656309, + "tokens_seen": 9699328 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.36840176582336426, + "objective/train/avg_token_score": 0.020729999989271164, + "objective/train/docs_used": 17861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.664196014404297, + "objective/train/original_loss": 7.552342891693115, + "objective/train/theoretical_loss": 6.9337544888949, + "objective/train/tokens_used": 9830400, + "objective/train/unlikelihood_loss": 0.06143795698881149, + "theoretical_loss": 6.9337544888949, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014880952380952382, + "loss": 2.7889, + "theoretical_loss": 6.9337544888949, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 2.387, + "theoretical_loss": 6.917004399797798, + "tokens_seen": 9961472 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001527777777777778, + "loss": 2.846, + "theoretical_loss": 6.900534068237688, + "tokens_seen": 10092544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015476190476190478, + "loss": 2.8487, + "theoretical_loss": 6.884335278496871, + "tokens_seen": 10223616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015674603174603175, + "loss": 2.8823, + "theoretical_loss": 6.868400157541997, + "tokens_seen": 10354688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 2.799, + "theoretical_loss": 6.85272115660663, + "tokens_seen": 10485760 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016071428571428573, + "loss": 2.9633, + "theoretical_loss": 6.8372910339797945, + "tokens_seen": 10616832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001626984126984127, + "loss": 2.5692, + "theoretical_loss": 6.8221028389077185, + "tokens_seen": 10747904 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016468253968253969, + "loss": 2.8166, + "theoretical_loss": 6.807149896524181, + "tokens_seen": 10878976 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016666666666666666, + "loss": 3.0421, + "theoretical_loss": 6.7924257937322245, + "tokens_seen": 11010048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016865079365079366, + "loss": 2.6027, + "theoretical_loss": 6.777924365966638, + "tokens_seen": 11141120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017063492063492064, + "loss": 2.9229, + "theoretical_loss": 6.763639684772625, + "tokens_seen": 11272192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017261904761904762, + "loss": 2.8946, + "theoretical_loss": 6.749566046141486, + "tokens_seen": 11403264 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.4332233667373657, + "objective/train/avg_token_score": 0.019017228856682777, + "objective/train/docs_used": 18493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.7806520462036133, + "objective/train/original_loss": 7.158781051635742, + "objective/train/theoretical_loss": 6.7426066439450905, + "objective/train/tokens_used": 11468800, + "objective/train/unlikelihood_loss": 0.05772477760910988, + "theoretical_loss": 6.7426066439450905, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001746031746031746, + "loss": 2.8532, + "theoretical_loss": 6.735697959549075, + "tokens_seen": 11534336 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001765873015873016, + "loss": 2.5473, + "theoretical_loss": 6.722030137647226, + "tokens_seen": 11665408 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017857142857142857, + "loss": 2.4778, + "theoretical_loss": 6.7085574865624125, + "tokens_seen": 11796480 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018055555555555555, + "loss": 2.9024, + "theoretical_loss": 6.695275096759559, + "tokens_seen": 11927552 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968252, + "loss": 2.7962, + "theoretical_loss": 6.682178234432274, + "tokens_seen": 12058624 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018452380952380953, + "loss": 2.6879, + "theoretical_loss": 6.669262333383815, + "tokens_seen": 12189696 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001865079365079365, + "loss": 2.9128, + "theoretical_loss": 6.656522987365879, + "tokens_seen": 12320768 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001884920634920635, + "loss": 2.4778, + "theoretical_loss": 6.643955942844831, + "tokens_seen": 12451840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 2.5956, + "theoretical_loss": 6.631557092167304, + "tokens_seen": 12582912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019246031746031748, + "loss": 2.8293, + "theoretical_loss": 6.619322467099223, + "tokens_seen": 12713984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019444444444444446, + "loss": 3.043, + "theoretical_loss": 6.607248232714213, + "tokens_seen": 12845056 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019642857142857144, + "loss": 2.7412, + "theoretical_loss": 6.59533068160918, + "tokens_seen": 12976128 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.4409824013710022, + "objective/train/avg_token_score": 0.007921183481812477, + "objective/train/docs_used": 19731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.8607468605041504, + "objective/train/original_loss": 5.793218612670898, + "objective/train/theoretical_loss": 6.583566228426414, + "objective/train/tokens_used": 13107200, + "objective/train/unlikelihood_loss": 0.14077132940292358, + "theoretical_loss": 6.583566228426414, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001984126984126984, + "loss": 2.5021, + "theoretical_loss": 6.583566228426414, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020039682539682542, + "loss": 2.5258, + "theoretical_loss": 6.571951404663098, + "tokens_seen": 13238272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002023809523809524, + "loss": 2.8582, + "theoretical_loss": 6.560482853750463, + "tokens_seen": 13369344 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020436507936507937, + "loss": 2.828, + "theoretical_loss": 6.549157326386091, + "tokens_seen": 13500416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920634, + "loss": 2.8331, + "theoretical_loss": 6.537971676104026, + "tokens_seen": 13631488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020833333333333335, + "loss": 2.0467, + "theoretical_loss": 6.5269228550684195, + "tokens_seen": 13762560 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021031746031746032, + "loss": 2.6811, + "theoretical_loss": 6.516007910077416, + "tokens_seen": 13893632 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002123015873015873, + "loss": 2.7855, + "theoretical_loss": 6.50522397876491, + "tokens_seen": 14024704 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021428571428571427, + "loss": 2.7877, + "theoretical_loss": 6.494568285988618, + "tokens_seen": 14155776 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021626984126984128, + "loss": 2.624, + "theoretical_loss": 6.484038140393699, + "tokens_seen": 14286848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021825396825396825, + "loss": 2.7998, + "theoretical_loss": 6.473630931141869, + "tokens_seen": 14417920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022023809523809523, + "loss": 2.71, + "theoretical_loss": 6.463344124796616, + "tokens_seen": 14548992 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002222222222222222, + "loss": 2.5073, + "theoretical_loss": 6.45317526235573, + "tokens_seen": 14680064 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.49395161867141724, + "objective/train/avg_token_score": 0.028753770515322685, + "objective/train/docs_used": 20428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 3.2851853370666504, + "objective/train/original_loss": 6.822603225708008, + "objective/train/theoretical_loss": 6.448134311315593, + "objective/train/tokens_used": 14745600, + "objective/train/unlikelihood_loss": 0.016451282426714897, + "theoretical_loss": 6.448134311315593, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002242063492063492, + "loss": 2.5962, + "theoretical_loss": 6.443121956422939, + "tokens_seen": 14811136 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022619047619047618, + "loss": 2.6017, + "theoretical_loss": 6.433181888510964, + "tokens_seen": 14942208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022817460317460316, + "loss": 2.7709, + "theoretical_loss": 6.4233528064687855, + "tokens_seen": 15073280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 2.5309, + "theoretical_loss": 6.413632522026391, + "tokens_seen": 15204352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023214285714285717, + "loss": 2.5149, + "theoretical_loss": 6.404018908450656, + "tokens_seen": 15335424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023412698412698414, + "loss": 2.5856, + "theoretical_loss": 6.394509898306452, + "tokens_seen": 15466496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023611111111111112, + "loss": 2.5106, + "theoretical_loss": 6.385103481317387, + "tokens_seen": 15597568 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 2.559, + "theoretical_loss": 6.375797702320966, + "tokens_seen": 15728640 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002400793650793651, + "loss": 2.2329, + "theoretical_loss": 6.366590659313248, + "tokens_seen": 15859712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024206349206349207, + "loss": 2.6909, + "theoretical_loss": 6.357480501578371, + "tokens_seen": 15990784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024404761904761905, + "loss": 2.6294, + "theoretical_loss": 6.348465427898629, + "tokens_seen": 16121856 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246031746031746, + "loss": 2.464, + "theoretical_loss": 6.33954368484097, + "tokens_seen": 16252928 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.39424487948417664, + "objective/train/avg_token_score": 0.013139091432094574, + "objective/train/docs_used": 21641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.531733751296997, + "objective/train/original_loss": 6.893299102783203, + "objective/train/theoretical_loss": 6.330713565116083, + "objective/train/tokens_used": 16384000, + "objective/train/unlikelihood_loss": 0.03236621245741844, + "theoretical_loss": 6.330713565116083, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248015873015873, + "loss": 2.6212, + "theoretical_loss": 6.330713565116083, + "tokens_seen": 16384000 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 2.7361, + "theoretical_loss": 6.32197340600647, + "tokens_seen": 16515072 + }, + { + "epoch": 0.01, + "learning_rate": 0.000251984126984127, + "loss": 2.4975, + "theoretical_loss": 6.313321587860021, + "tokens_seen": 16646144 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025396825396825396, + "loss": 2.75, + "theoretical_loss": 6.304756532645939, + "tokens_seen": 16777216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025595238095238096, + "loss": 2.38, + "theoretical_loss": 6.296276702569918, + "tokens_seen": 16908288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025793650793650796, + "loss": 2.4812, + "theoretical_loss": 6.28788059874573, + "tokens_seen": 17039360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025992063492063497, + "loss": 2.5431, + "theoretical_loss": 6.279566759920507, + "tokens_seen": 17170432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002619047619047619, + "loss": 2.9019, + "theoretical_loss": 6.271333761251142, + "tokens_seen": 17301504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002638888888888889, + "loss": 2.454, + "theoretical_loss": 6.2631802131294085, + "tokens_seen": 17432576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026587301587301587, + "loss": 2.6175, + "theoretical_loss": 6.255104760053497, + "tokens_seen": 17563648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026785714285714287, + "loss": 2.6316, + "theoretical_loss": 6.247106079543801, + "tokens_seen": 17694720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002698412698412698, + "loss": 2.4119, + "theoretical_loss": 6.239182881100916, + "tokens_seen": 17825792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002718253968253968, + "loss": 2.7574, + "theoretical_loss": 6.231333905203899, + "tokens_seen": 17956864 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.3819648027420044, + "objective/train/avg_token_score": 0.008278077468276024, + "objective/train/docs_used": 22253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.420483112335205, + "objective/train/original_loss": 6.370981693267822, + "objective/train/theoretical_loss": 6.227436865342643, + "objective/train/tokens_used": 18022400, + "objective/train/unlikelihood_loss": 0.04339008778333664, + "theoretical_loss": 6.227436865342643, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002738095238095238, + "loss": 2.6268, + "theoretical_loss": 6.223557922346955, + "tokens_seen": 18087936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027579365079365083, + "loss": 2.5336, + "theoretical_loss": 6.215853732112821, + "tokens_seen": 18219008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002777777777777778, + "loss": 2.3994, + "theoretical_loss": 6.208220162281178, + "tokens_seen": 18350080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002797619047619048, + "loss": 2.3323, + "theoretical_loss": 6.20065606797053, + "tokens_seen": 18481152 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028174603174603173, + "loss": 2.5059, + "theoretical_loss": 6.1931603308120975, + "tokens_seen": 18612224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028373015873015873, + "loss": 2.28, + "theoretical_loss": 6.185731858154261, + "tokens_seen": 18743296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002857142857142857, + "loss": 2.4878, + "theoretical_loss": 6.17836958229627, + "tokens_seen": 18874368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002876984126984127, + "loss": 2.4956, + "theoretical_loss": 6.171072459749913, + "tokens_seen": 19005440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002896825396825397, + "loss": 2.2678, + "theoretical_loss": 6.163839470527964, + "tokens_seen": 19136512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002916666666666667, + "loss": 2.1494, + "theoretical_loss": 6.156669617458243, + "tokens_seen": 19267584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002936507936507937, + "loss": 2.3674, + "theoretical_loss": 6.149561925522211, + "tokens_seen": 19398656 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029563492063492064, + "loss": 2.3958, + "theoretical_loss": 6.142515441217064, + "tokens_seen": 19529728 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4558895528316498, + "objective/train/avg_token_score": 0.004920742008835077, + "objective/train/docs_used": 23612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.9035089015960693, + "objective/train/original_loss": 6.259584426879883, + "objective/train/theoretical_loss": 6.135529231940326, + "objective/train/tokens_used": 19660800, + "objective/train/unlikelihood_loss": 0.0633547455072403, + "theoretical_loss": 6.135529231940326, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029761904761904765, + "loss": 2.5385, + "theoretical_loss": 6.135529231940326, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002996031746031746, + "loss": 2.3807, + "theoretical_loss": 6.128602385396022, + "tokens_seen": 19791872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003015873015873016, + "loss": 2.0421, + "theoretical_loss": 6.121734009021521, + "tokens_seen": 19922944 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030357142857142855, + "loss": 2.5316, + "theoretical_loss": 6.114923229434213, + "tokens_seen": 20054016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003055555555555556, + "loss": 2.4607, + "theoretical_loss": 6.108169191897195, + "tokens_seen": 20185088 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030753968253968255, + "loss": 2.5377, + "theoretical_loss": 6.101471059803204, + "tokens_seen": 20316160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030952380952380956, + "loss": 2.7593, + "theoretical_loss": 6.094828014176053, + "tokens_seen": 20447232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003115079365079365, + "loss": 2.6748, + "theoretical_loss": 6.088239253188885, + "tokens_seen": 20578304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003134920634920635, + "loss": 2.387, + "theoretical_loss": 6.0817039916985465, + "tokens_seen": 20709376 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031547619047619046, + "loss": 2.3745, + "theoretical_loss": 6.075221460795472, + "tokens_seen": 20840448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031746031746031746, + "loss": 2.3548, + "theoretical_loss": 6.068790907368448, + "tokens_seen": 20971520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003194444444444444, + "loss": 2.5693, + "theoretical_loss": 6.062411593683687, + "tokens_seen": 21102592 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032142857142857147, + "loss": 2.754, + "theoretical_loss": 6.056082796977648, + "tokens_seen": 21233664 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.5116080045700073, + "objective/train/avg_token_score": 0.0047623817808926105, + "objective/train/docs_used": 24330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 3.2419803142547607, + "objective/train/original_loss": 5.9219136238098145, + "objective/train/theoretical_loss": 6.052937120624258, + "objective/train/tokens_used": 21299200, + "objective/train/unlikelihood_loss": 0.10302968323230743, + "theoretical_loss": 6.052937120624258, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003234126984126984, + "loss": 2.6761, + "theoretical_loss": 6.049803809063083, + "tokens_seen": 21364736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003253968253968254, + "loss": 2.3248, + "theoretical_loss": 6.04357393594778, + "tokens_seen": 21495808 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032738095238095237, + "loss": 2.5682, + "theoretical_loss": 6.037392497465552, + "tokens_seen": 21626880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032936507936507937, + "loss": 2.2946, + "theoretical_loss": 6.031258826918979, + "tokens_seen": 21757952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003313492063492063, + "loss": 2.6749, + "theoretical_loss": 6.025172270733464, + "tokens_seen": 21889024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003333333333333333, + "loss": 2.6852, + "theoretical_loss": 6.0191321881221995, + "tokens_seen": 22020096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003353174603174603, + "loss": 2.5813, + "theoretical_loss": 6.01313795076161, + "tokens_seen": 22151168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033730158730158733, + "loss": 2.6132, + "theoretical_loss": 6.007188942476907, + "tokens_seen": 22282240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033928571428571433, + "loss": 2.1199, + "theoretical_loss": 6.001284558937368, + "tokens_seen": 22413312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003412698412698413, + "loss": 2.5638, + "theoretical_loss": 5.995424207360987, + "tokens_seen": 22544384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003432539682539683, + "loss": 2.4583, + "theoretical_loss": 5.989607306228168, + "tokens_seen": 22675456 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034523809523809523, + "loss": 2.6098, + "theoretical_loss": 5.983833285004112, + "tokens_seen": 22806528 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.374450147151947, + "objective/train/avg_token_score": 0.010239812545478344, + "objective/train/docs_used": 24983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.322624921798706, + "objective/train/original_loss": 6.397256374359131, + "objective/train/theoretical_loss": 5.978101583869607, + "objective/train/tokens_used": 22937600, + "objective/train/unlikelihood_loss": 0.04547956585884094, + "theoretical_loss": 5.978101583869607, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034722222222222224, + "loss": 2.6236, + "theoretical_loss": 5.978101583869607, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003492063492063492, + "loss": 2.2842, + "theoretical_loss": 5.972411653459913, + "tokens_seen": 23068672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003511904761904762, + "loss": 2.4053, + "theoretical_loss": 5.966762954611432, + "tokens_seen": 23199744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003531746031746032, + "loss": 2.4468, + "theoretical_loss": 5.961154958115937, + "tokens_seen": 23330816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003551587301587302, + "loss": 2.7427, + "theoretical_loss": 5.955587144482044, + "tokens_seen": 23461888 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035714285714285714, + "loss": 2.6201, + "theoretical_loss": 5.950059003703704, + "tokens_seen": 23592960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035912698412698415, + "loss": 2.6046, + "theoretical_loss": 5.944570035035458, + "tokens_seen": 23724032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003611111111111111, + "loss": 2.4813, + "theoretical_loss": 5.939119746774228, + "tokens_seen": 23855104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003630952380952381, + "loss": 2.464, + "theoretical_loss": 5.933707656047414, + "tokens_seen": 23986176 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036507936507936505, + "loss": 2.5273, + "theoretical_loss": 5.928333288607086, + "tokens_seen": 24117248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036706349206349205, + "loss": 2.6318, + "theoretical_loss": 5.92299617863006, + "tokens_seen": 24248320 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036904761904761905, + "loss": 2.5566, + "theoretical_loss": 5.9176958685236585, + "tokens_seen": 24379392 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037103174603174606, + "loss": 2.3378, + "theoretical_loss": 5.912431908736972, + "tokens_seen": 24510464 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.3531891405582428, + "objective/train/avg_token_score": 0.034930985420942307, + "objective/train/docs_used": 26439, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0727763175964355, + "objective/train/original_loss": 6.117815971374512, + "objective/train/theoretical_loss": 5.909813421882534, + "objective/train/tokens_used": 24576000, + "objective/train/unlikelihood_loss": 0.0909704715013504, + "theoretical_loss": 5.909813421882534, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000373015873015873, + "loss": 2.2126, + "theoretical_loss": 5.907203857577422, + "tokens_seen": 24641536 + }, + { + "epoch": 0.01, + "learning_rate": 0.000375, + "loss": 2.7691, + "theoretical_loss": 5.902011281032472, + "tokens_seen": 24772608 + }, + { + "epoch": 0.01, + "learning_rate": 0.000376984126984127, + "loss": 2.3763, + "theoretical_loss": 5.896853752596286, + "tokens_seen": 24903680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037896825396825396, + "loss": 2.1913, + "theoretical_loss": 5.891730853101199, + "tokens_seen": 25034752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038095238095238096, + "loss": 2.6541, + "theoretical_loss": 5.8866421705538325, + "tokens_seen": 25165824 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038293650793650797, + "loss": 2.3709, + "theoretical_loss": 5.881587299975694, + "tokens_seen": 25296896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038492063492063497, + "loss": 2.0019, + "theoretical_loss": 5.876565843248124, + "tokens_seen": 25427968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003869047619047619, + "loss": 2.3081, + "theoretical_loss": 5.871577408961457, + "tokens_seen": 25559040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003888888888888889, + "loss": 2.1361, + "theoretical_loss": 5.866621612268246, + "tokens_seen": 25690112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039087301587301587, + "loss": 2.4905, + "theoretical_loss": 5.8616980747404295, + "tokens_seen": 25821184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003928571428571429, + "loss": 2.2697, + "theoretical_loss": 5.856806424230314, + "tokens_seen": 25952256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003948412698412698, + "loss": 2.3797, + "theoretical_loss": 5.851946294735258, + "tokens_seen": 26083328 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.48717010021209717, + "objective/train/avg_token_score": 0.020001767203211784, + "objective/train/docs_used": 27060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.8721370697021484, + "objective/train/original_loss": 5.758991718292236, + "objective/train/theoretical_loss": 5.8471173262659235, + "objective/train/tokens_used": 26214400, + "objective/train/unlikelihood_loss": 0.08089856803417206, + "theoretical_loss": 5.8471173262659235, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003968253968253968, + "loss": 2.4683, + "theoretical_loss": 5.8471173262659235, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039880952380952383, + "loss": 2.3909, + "theoretical_loss": 5.842319164718004, + "tokens_seen": 26345472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040079365079365083, + "loss": 2.3413, + "theoretical_loss": 5.83755146174731, + "tokens_seen": 26476544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004027777777777778, + "loss": 2.6428, + "theoretical_loss": 5.832813874648102, + "tokens_seen": 26607616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 2.5088, + "theoretical_loss": 5.828106066234588, + "tokens_seen": 26738688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040674603174603173, + "loss": 2.5131, + "theoretical_loss": 5.823427704725473, + "tokens_seen": 26869760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040873015873015874, + "loss": 2.8614, + "theoretical_loss": 5.818778463631473, + "tokens_seen": 27000832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004107142857142857, + "loss": 2.3157, + "theoretical_loss": 5.8141580216457065, + "tokens_seen": 27131904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004126984126984127, + "loss": 2.4281, + "theoretical_loss": 5.809566062536868, + "tokens_seen": 27262976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004146825396825397, + "loss": 2.6006, + "theoretical_loss": 5.805002275045111, + "tokens_seen": 27394048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004166666666666667, + "loss": 2.2208, + "theoretical_loss": 5.800466352780546, + "tokens_seen": 27525120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004186507936507937, + "loss": 2.4363, + "theoretical_loss": 5.795957994124291, + "tokens_seen": 27656192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 2.5718, + "theoretical_loss": 5.791476902131985, + "tokens_seen": 27787264 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.39797165989875793, + "objective/train/avg_token_score": 0.010648269206285477, + "objective/train/docs_used": 28453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.4116973876953125, + "objective/train/original_loss": 6.296491622924805, + "objective/train/theoretical_loss": 5.789246489594688, + "objective/train/tokens_used": 27852800, + "objective/train/unlikelihood_loss": 0.0855846107006073, + "theoretical_loss": 5.789246489594688, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042261904761904765, + "loss": 2.3402, + "theoretical_loss": 5.787022784439701, + "tokens_seen": 27918336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004246031746031746, + "loss": 2.5929, + "theoretical_loss": 5.782595353172176, + "tokens_seen": 28049408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004265873015873016, + "loss": 2.671, + "theoretical_loss": 5.778194324853311, + "tokens_seen": 28180480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042857142857142855, + "loss": 2.5018, + "theoretical_loss": 5.773819420318858, + "tokens_seen": 28311552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004305555555555556, + "loss": 2.2753, + "theoretical_loss": 5.769470364631225, + "tokens_seen": 28442624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043253968253968256, + "loss": 2.0779, + "theoretical_loss": 5.765146886996363, + "tokens_seen": 28573696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043452380952380956, + "loss": 2.1053, + "theoretical_loss": 5.760848720682651, + "tokens_seen": 28704768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 2.3278, + "theoretical_loss": 5.756575602941732, + "tokens_seen": 28835840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004384920634920635, + "loss": 2.4976, + "theoretical_loss": 5.752327274931249, + "tokens_seen": 28966912 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044047619047619046, + "loss": 2.2979, + "theoretical_loss": 5.7481034816394105, + "tokens_seen": 29097984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044246031746031746, + "loss": 2.4162, + "theoretical_loss": 5.74390397181136, + "tokens_seen": 29229056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004444444444444444, + "loss": 2.4311, + "theoretical_loss": 5.739728497877267, + "tokens_seen": 29360128 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.28812316060066223, + "objective/train/avg_token_score": 0.02659354917705059, + "objective/train/docs_used": 29115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.703152060508728, + "objective/train/original_loss": 6.603550434112549, + "objective/train/theoretical_loss": 5.7355768158821245, + "objective/train/tokens_used": 29491200, + "objective/train/unlikelihood_loss": 0.04069405049085617, + "theoretical_loss": 5.7355768158821245, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044642857142857147, + "loss": 2.4414, + "theoretical_loss": 5.7355768158821245, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004484126984126984, + "loss": 2.4985, + "theoretical_loss": 5.731448685417178, + "tokens_seen": 29622272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004503968253968254, + "loss": 2.2532, + "theoretical_loss": 5.7273438695529535, + "tokens_seen": 29753344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 2.4096, + "theoretical_loss": 5.7232621347738455, + "tokens_seen": 29884416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045436507936507937, + "loss": 2.1835, + "theoretical_loss": 5.719203250914208, + "tokens_seen": 30015488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004563492063492063, + "loss": 2.4457, + "theoretical_loss": 5.715166991095922, + "tokens_seen": 30146560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004583333333333333, + "loss": 2.598, + "theoretical_loss": 5.71115313166738, + "tokens_seen": 30277632 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 2.5442, + "theoretical_loss": 5.707161452143879, + "tokens_seen": 30408704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046230158730158733, + "loss": 2.2213, + "theoretical_loss": 5.7031917351493515, + "tokens_seen": 30539776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046428571428571433, + "loss": 2.178, + "theoretical_loss": 5.699243766359421, + "tokens_seen": 30670848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004662698412698413, + "loss": 2.4453, + "theoretical_loss": 5.695317334445736, + "tokens_seen": 30801920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 2.4695, + "theoretical_loss": 5.691412231021549, + "tokens_seen": 30932992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047023809523809523, + "loss": 2.4932, + "theoretical_loss": 5.687528250588518, + "tokens_seen": 31064064 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.21346530318260193, + "objective/train/avg_token_score": 0.01769358292222023, + "objective/train/docs_used": 30152, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2651164531707764, + "objective/train/original_loss": 6.490805149078369, + "objective/train/theoretical_loss": 5.6855941180524265, + "objective/train/tokens_used": 31129600, + "objective/train/unlikelihood_loss": 0.0511309988796711, + "theoretical_loss": 5.6855941180524265, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047222222222222224, + "loss": 2.1794, + "theoretical_loss": 5.683665190484683, + "tokens_seen": 31195136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004742063492063492, + "loss": 2.251, + "theoretical_loss": 5.679822850833591, + "tokens_seen": 31326208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 2.3855, + "theoretical_loss": 5.676001034494554, + "tokens_seen": 31457280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004781746031746032, + "loss": 2.352, + "theoretical_loss": 5.672199547013983, + "tokens_seen": 31588352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004801587301587302, + "loss": 2.5004, + "theoretical_loss": 5.6684181965778, + "tokens_seen": 31719424 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048214285714285715, + "loss": 2.3015, + "theoretical_loss": 5.6646567939648715, + "tokens_seen": 31850496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048412698412698415, + "loss": 2.358, + "theoretical_loss": 5.660915152501465, + "tokens_seen": 31981568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004861111111111111, + "loss": 2.496, + "theoretical_loss": 5.657193088016677, + "tokens_seen": 32112640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004880952380952381, + "loss": 2.3284, + "theoretical_loss": 5.653490418798825, + "tokens_seen": 32243712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004900793650793651, + "loss": 2.2022, + "theoretical_loss": 5.649806965552774, + "tokens_seen": 32374784 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492063492063492, + "loss": 2.1381, + "theoretical_loss": 5.6461425513581665, + "tokens_seen": 32505856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004940476190476191, + "loss": 2.6351, + "theoretical_loss": 5.6424970016285485, + "tokens_seen": 32636928 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4376832842826843, + "objective/train/avg_token_score": 0.009709020145237446, + "objective/train/docs_used": 30758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.5283288955688477, + "objective/train/original_loss": 5.68566370010376, + "objective/train/theoretical_loss": 5.638870144071353, + "objective/train/tokens_used": 32768000, + "objective/train/unlikelihood_loss": 0.07930290699005127, + "theoretical_loss": 5.638870144071353, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496031746031746, + "loss": 2.3134, + "theoretical_loss": 5.638870144071353, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498015873015873, + "loss": 2.3275, + "theoretical_loss": 5.635261808648728, + "tokens_seen": 32899072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 2.2282, + "theoretical_loss": 5.631671827539186, + "tokens_seen": 33030144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999799398194584, + "loss": 2.256, + "theoretical_loss": 5.628100035100061, + "tokens_seen": 33161216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999598796389167, + "loss": 2.2104, + "theoretical_loss": 5.6245462678307385, + "tokens_seen": 33292288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999398194583751, + "loss": 2.2591, + "theoretical_loss": 5.621010364336651, + "tokens_seen": 33423360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999197592778335, + "loss": 2.4118, + "theoretical_loss": 5.61749216529402, + "tokens_seen": 33554432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998996990972919, + "loss": 2.2046, + "theoretical_loss": 5.61399151341532, + "tokens_seen": 33685504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998796389167503, + "loss": 2.3628, + "theoretical_loss": 5.610508253415453, + "tokens_seen": 33816576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998595787362087, + "loss": 2.3158, + "theoretical_loss": 5.6070422319786095, + "tokens_seen": 33947648 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499839518555667, + "loss": 2.2646, + "theoretical_loss": 5.603593297725807, + "tokens_seen": 34078720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998194583751254, + "loss": 2.4531, + "theoretical_loss": 5.600161301183084, + "tokens_seen": 34209792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997993981945837, + "loss": 2.3033, + "theoretical_loss": 5.596746094750342, + "tokens_seen": 34340864 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.31781524419784546, + "objective/train/avg_token_score": 0.06681742519140244, + "objective/train/docs_used": 31355, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7797188758850098, + "objective/train/original_loss": 5.599770545959473, + "objective/train/theoretical_loss": 5.595044742205037, + "objective/train/tokens_used": 34406400, + "objective/train/unlikelihood_loss": 0.1539888083934784, + "theoretical_loss": 5.595044742205037, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997793380140421, + "loss": 2.0253, + "theoretical_loss": 5.5933475326707995, + "tokens_seen": 34471936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997592778335005, + "loss": 2.3513, + "theoretical_loss": 5.589965471001077, + "tokens_seen": 34603008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997392176529588, + "loss": 2.385, + "theoretical_loss": 5.586599767581859, + "tokens_seen": 34734080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997191574724173, + "loss": 2.4408, + "theoretical_loss": 5.583250282009159, + "tokens_seen": 34865152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996990972918757, + "loss": 2.2335, + "theoretical_loss": 5.579916875606134, + "tokens_seen": 34996224 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499679037111334, + "loss": 2.2096, + "theoretical_loss": 5.576599411395472, + "tokens_seen": 35127296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996589769307924, + "loss": 2.5149, + "theoretical_loss": 5.5732977540723105, + "tokens_seen": 35258368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996389167502507, + "loss": 2.1646, + "theoretical_loss": 5.570011769977693, + "tokens_seen": 35389440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996188565697091, + "loss": 2.2061, + "theoretical_loss": 5.566741327072535, + "tokens_seen": 35520512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995987963891675, + "loss": 2.462, + "theoretical_loss": 5.563486294912105, + "tokens_seen": 35651584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995787362086258, + "loss": 2.1712, + "theoretical_loss": 5.560246544620993, + "tokens_seen": 35782656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995586760280842, + "loss": 2.4396, + "theoretical_loss": 5.557021948868571, + "tokens_seen": 35913728 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.37939882278442383, + "objective/train/avg_token_score": 0.02247542515397072, + "objective/train/docs_used": 32385, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.2949864864349365, + "objective/train/original_loss": 5.736522674560547, + "objective/train/theoretical_loss": 5.553812381844907, + "objective/train/tokens_used": 36044800, + "objective/train/unlikelihood_loss": 0.08487144112586975, + "theoretical_loss": 5.553812381844907, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995386158475427, + "loss": 2.2289, + "theoretical_loss": 5.553812381844907, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499518555667001, + "loss": 2.3559, + "theoretical_loss": 5.550617719237167, + "tokens_seen": 36175872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994984954864594, + "loss": 2.1262, + "theoretical_loss": 5.547437838206435, + "tokens_seen": 36306944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994784353059178, + "loss": 2.2879, + "theoretical_loss": 5.544272617365014, + "tokens_seen": 36438016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994583751253761, + "loss": 2.2899, + "theoretical_loss": 5.541121936754111, + "tokens_seen": 36569088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994383149448345, + "loss": 2.1249, + "theoretical_loss": 5.537985677821986, + "tokens_seen": 36700160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994182547642928, + "loss": 2.5925, + "theoretical_loss": 5.5348637234024824, + "tokens_seen": 36831232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993981945837512, + "loss": 2.3988, + "theoretical_loss": 5.5317559576939725, + "tokens_seen": 36962304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993781344032096, + "loss": 2.1127, + "theoretical_loss": 5.5286622662386975, + "tokens_seen": 37093376 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499358074222668, + "loss": 2.2598, + "theoretical_loss": 5.525582535902489, + "tokens_seen": 37224448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993380140421264, + "loss": 2.2746, + "theoretical_loss": 5.52251665485486, + "tokens_seen": 37355520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993179538615848, + "loss": 2.0758, + "theoretical_loss": 5.519464512549478, + "tokens_seen": 37486592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992978936810431, + "loss": 2.0689, + "theoretical_loss": 5.516425999704987, + "tokens_seen": 37617664 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4496578574180603, + "objective/train/avg_token_score": 0.006494433153420687, + "objective/train/docs_used": 32883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.570333480834961, + "objective/train/original_loss": 5.788158416748047, + "objective/train/theoretical_loss": 5.514911820521407, + "objective/train/tokens_used": 37683200, + "objective/train/unlikelihood_loss": 0.06332194805145264, + "theoretical_loss": 5.514911820521407, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992778335005015, + "loss": 2.2661, + "theoretical_loss": 5.5134010082861895, + "tokens_seen": 37748736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992577733199598, + "loss": 2.4036, + "theoretical_loss": 5.51038943148556, + "tokens_seen": 37879808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992377131394183, + "loss": 2.6818, + "theoretical_loss": 5.50739116370511, + "tokens_seen": 38010880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992176529588767, + "loss": 2.1309, + "theoretical_loss": 5.5044061005385725, + "tokens_seen": 38141952 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499197592778335, + "loss": 2.3905, + "theoretical_loss": 5.501434138753918, + "tokens_seen": 38273024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991775325977934, + "loss": 2.5159, + "theoretical_loss": 5.498475176276176, + "tokens_seen": 38404096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991574724172518, + "loss": 2.6962, + "theoretical_loss": 5.495529112170568, + "tokens_seen": 38535168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991374122367101, + "loss": 2.1093, + "theoretical_loss": 5.492595846625951, + "tokens_seen": 38666240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991173520561685, + "loss": 2.4076, + "theoretical_loss": 5.489675280938547, + "tokens_seen": 38797312 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499097291875627, + "loss": 2.3561, + "theoretical_loss": 5.486767317495966, + "tokens_seen": 38928384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990772316950853, + "loss": 2.4408, + "theoretical_loss": 5.483871859761511, + "tokens_seen": 39059456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990571715145437, + "loss": 2.1575, + "theoretical_loss": 5.480988812258763, + "tokens_seen": 39190528 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.41135141253471375, + "objective/train/avg_token_score": 0.026532353833317757, + "objective/train/docs_used": 34166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.235038995742798, + "objective/train/original_loss": 5.851961612701416, + "objective/train/theoretical_loss": 5.478118080556438, + "objective/train/tokens_used": 39321600, + "objective/train/unlikelihood_loss": 0.04781878739595413, + "theoretical_loss": 5.478118080556438, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499037111334002, + "loss": 2.1814, + "theoretical_loss": 5.478118080556438, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990170511534604, + "loss": 2.2678, + "theoretical_loss": 5.475259571253502, + "tokens_seen": 39452672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989969909729188, + "loss": 2.4555, + "theoretical_loss": 5.4724131919645576, + "tokens_seen": 39583744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989769307923771, + "loss": 2.2421, + "theoretical_loss": 5.4695788513054815, + "tokens_seen": 39714816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989568706118355, + "loss": 2.2966, + "theoretical_loss": 5.466756458879306, + "tokens_seen": 39845888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989368104312939, + "loss": 2.578, + "theoretical_loss": 5.463945925262355, + "tokens_seen": 39976960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989167502507523, + "loss": 1.9973, + "theoretical_loss": 5.461147161990611, + "tokens_seen": 40108032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988966900702107, + "loss": 2.4388, + "theoretical_loss": 5.458360081546321, + "tokens_seen": 40239104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988766298896691, + "loss": 2.4184, + "theoretical_loss": 5.455584597344835, + "tokens_seen": 40370176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988565697091274, + "loss": 2.1967, + "theoretical_loss": 5.452820623721662, + "tokens_seen": 40501248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988365095285858, + "loss": 2.3451, + "theoretical_loss": 5.450068075919752, + "tokens_seen": 40632320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988164493480441, + "loss": 2.0896, + "theoretical_loss": 5.447326870076996, + "tokens_seen": 40763392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987963891675025, + "loss": 2.1861, + "theoretical_loss": 5.444596923213931, + "tokens_seen": 40894464 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.33492180705070496, + "objective/train/avg_token_score": 0.03136708214879036, + "objective/train/docs_used": 34866, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9246761798858643, + "objective/train/original_loss": 5.993686199188232, + "objective/train/theoretical_loss": 5.443236146207074, + "objective/train/tokens_used": 40960000, + "objective/train/unlikelihood_loss": 0.09010782837867737, + "theoretical_loss": 5.443236146207074, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987763289869609, + "loss": 2.2846, + "theoretical_loss": 5.441878153221662, + "tokens_seen": 41025536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987562688064192, + "loss": 2.4934, + "theoretical_loss": 5.439170478849976, + "tokens_seen": 41156608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987362086258777, + "loss": 1.8671, + "theoretical_loss": 5.4364738196956655, + "tokens_seen": 41287680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987161484453361, + "loss": 2.2663, + "theoretical_loss": 5.433788096191039, + "tokens_seen": 41418752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986960882647944, + "loss": 2.3139, + "theoretical_loss": 5.4311132295926345, + "tokens_seen": 41549824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986760280842528, + "loss": 2.5888, + "theoretical_loss": 5.428449141970107, + "tokens_seen": 41680896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986559679037111, + "loss": 2.1469, + "theoretical_loss": 5.42579575619531, + "tokens_seen": 41811968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986359077231695, + "loss": 2.3618, + "theoretical_loss": 5.423152995931552, + "tokens_seen": 41943040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986158475426279, + "loss": 2.3092, + "theoretical_loss": 5.420520785623031, + "tokens_seen": 42074112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985957873620862, + "loss": 2.3179, + "theoretical_loss": 5.417899050484451, + "tokens_seen": 42205184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985757271815446, + "loss": 2.4495, + "theoretical_loss": 5.415287716490787, + "tokens_seen": 42336256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985556670010031, + "loss": 2.2482, + "theoretical_loss": 5.412686710367245, + "tokens_seen": 42467328 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.3299120366573334, + "objective/train/avg_token_score": 0.04832848906517029, + "objective/train/docs_used": 35868, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8417558670043945, + "objective/train/original_loss": 5.99447774887085, + "objective/train/theoretical_loss": 5.410095959579362, + "objective/train/tokens_used": 42598400, + "objective/train/unlikelihood_loss": 0.052462801337242126, + "theoretical_loss": 5.410095959579362, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985356068204614, + "loss": 1.9885, + "theoretical_loss": 5.410095959579362, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985155466399198, + "loss": 2.4452, + "theoretical_loss": 5.407515392323276, + "tokens_seen": 42729472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984954864593782, + "loss": 2.3031, + "theoretical_loss": 5.404944937516161, + "tokens_seen": 42860544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984754262788365, + "loss": 2.2327, + "theoretical_loss": 5.402384524786797, + "tokens_seen": 42991616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984553660982949, + "loss": 1.8473, + "theoretical_loss": 5.399834084466306, + "tokens_seen": 43122688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984353059177532, + "loss": 2.2829, + "theoretical_loss": 5.397293547579041, + "tokens_seen": 43253760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984152457372116, + "loss": 2.2311, + "theoretical_loss": 5.394762845833601, + "tokens_seen": 43384832 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049839518555667, + "loss": 2.3256, + "theoretical_loss": 5.392241911614005, + "tokens_seen": 43515904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983751253761284, + "loss": 2.2566, + "theoretical_loss": 5.389730677971002, + "tokens_seen": 43646976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983550651955868, + "loss": 2.5088, + "theoretical_loss": 5.387229078613521, + "tokens_seen": 43778048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983350050150452, + "loss": 2.1468, + "theoretical_loss": 5.384737047900243, + "tokens_seen": 43909120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983149448345035, + "loss": 2.3067, + "theoretical_loss": 5.382254520831328, + "tokens_seen": 44040192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982948846539619, + "loss": 1.9734, + "theoretical_loss": 5.379781433040252, + "tokens_seen": 44171264 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.345246821641922, + "objective/train/avg_token_score": 0.005966670345515013, + "objective/train/docs_used": 36544, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9924170970916748, + "objective/train/original_loss": 5.951955795288086, + "objective/train/theoretical_loss": 5.378548408930558, + "objective/train/tokens_used": 44236800, + "objective/train/unlikelihood_loss": 0.06286505609750748, + "theoretical_loss": 5.378548408930558, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982748244734202, + "loss": 2.169, + "theoretical_loss": 5.377317720785777, + "tokens_seen": 44302336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982547642928786, + "loss": 2.5078, + "theoretical_loss": 5.374863320944057, + "tokens_seen": 44433408 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498234704112337, + "loss": 2.2855, + "theoretical_loss": 5.372418171000847, + "tokens_seen": 44564480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982146439317953, + "loss": 2.2421, + "theoretical_loss": 5.369982209043851, + "tokens_seen": 44695552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981945837512538, + "loss": 2.4581, + "theoretical_loss": 5.367555373755179, + "tokens_seen": 44826624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981745235707122, + "loss": 2.3194, + "theoretical_loss": 5.365137604403923, + "tokens_seen": 44957696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981544633901705, + "loss": 2.1544, + "theoretical_loss": 5.362728840838843, + "tokens_seen": 45088768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981344032096289, + "loss": 2.0627, + "theoretical_loss": 5.360329023481169, + "tokens_seen": 45219840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981143430290873, + "loss": 2.2282, + "theoretical_loss": 5.357938093317518, + "tokens_seen": 45350912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980942828485456, + "loss": 2.1852, + "theoretical_loss": 5.355555991892905, + "tokens_seen": 45481984 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498074222668004, + "loss": 2.147, + "theoretical_loss": 5.353182661303873, + "tokens_seen": 45613056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980541624874623, + "loss": 2.3338, + "theoretical_loss": 5.350818044191721, + "tokens_seen": 45744128 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4638318717479706, + "objective/train/avg_token_score": 0.017905525863170624, + "objective/train/docs_used": 37915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.485879421234131, + "objective/train/original_loss": 5.818847179412842, + "objective/train/theoretical_loss": 5.348462083735834, + "objective/train/tokens_used": 45875200, + "objective/train/unlikelihood_loss": 0.05798850208520889, + "theoretical_loss": 5.348462083735834, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980341023069207, + "loss": 2.2988, + "theoretical_loss": 5.348462083735834, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980140421263792, + "loss": 2.3518, + "theoretical_loss": 5.346114723647119, + "tokens_seen": 46006272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979939819458375, + "loss": 2.4436, + "theoretical_loss": 5.343775908161532, + "tokens_seen": 46137344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979739217652959, + "loss": 2.5148, + "theoretical_loss": 5.341445582033705, + "tokens_seen": 46268416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979538615847543, + "loss": 2.2889, + "theoretical_loss": 5.339123690530673, + "tokens_seen": 46399488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979338014042126, + "loss": 2.4053, + "theoretical_loss": 5.336810179425685, + "tokens_seen": 46530560 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497913741223671, + "loss": 2.5862, + "theoretical_loss": 5.334504994992115, + "tokens_seen": 46661632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978936810431293, + "loss": 2.2837, + "theoretical_loss": 5.332208083997459, + "tokens_seen": 46792704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978736208625877, + "loss": 2.2526, + "theoretical_loss": 5.329919393697422, + "tokens_seen": 46923776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978535606820461, + "loss": 2.2839, + "theoretical_loss": 5.327638871830089, + "tokens_seen": 47054848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978335005015045, + "loss": 2.1244, + "theoretical_loss": 5.32536646661019, + "tokens_seen": 47185920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978134403209629, + "loss": 2.3265, + "theoretical_loss": 5.323102126723439, + "tokens_seen": 47316992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977933801404213, + "loss": 2.1693, + "theoretical_loss": 5.320845801320959, + "tokens_seen": 47448064 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.5400782227516174, + "objective/train/avg_token_score": 0.00798075646162033, + "objective/train/docs_used": 38651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.944997549057007, + "objective/train/original_loss": 5.1642985343933105, + "objective/train/theoretical_loss": 5.319720628286955, + "objective/train/tokens_used": 47513600, + "objective/train/unlikelihood_loss": 0.09301882237195969, + "theoretical_loss": 5.319720628286955, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977733199598796, + "loss": 2.4382, + "theoretical_loss": 5.318597440013795, + "tokens_seen": 47579136 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497753259779338, + "loss": 2.4145, + "theoretical_loss": 5.316356992867491, + "tokens_seen": 47710208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977331995987965, + "loss": 2.3468, + "theoretical_loss": 5.314124410396767, + "tokens_seen": 47841280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977131394182548, + "loss": 1.8894, + "theoretical_loss": 5.311899643560251, + "tokens_seen": 47972352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976930792377132, + "loss": 2.3309, + "theoretical_loss": 5.3096826437553, + "tokens_seen": 48103424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976730190571715, + "loss": 2.1528, + "theoretical_loss": 5.3074733628129005, + "tokens_seen": 48234496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976529588766299, + "loss": 2.1967, + "theoretical_loss": 5.305271752992619, + "tokens_seen": 48365568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976328986960883, + "loss": 2.3363, + "theoretical_loss": 5.303077766977653, + "tokens_seen": 48496640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976128385155466, + "loss": 2.1614, + "theoretical_loss": 5.300891357869929, + "tokens_seen": 48627712 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497592778335005, + "loss": 2.4155, + "theoretical_loss": 5.298712479185288, + "tokens_seen": 48758784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975727181544635, + "loss": 2.1652, + "theoretical_loss": 5.296541084848727, + "tokens_seen": 48889856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975526579739218, + "loss": 2.2516, + "theoretical_loss": 5.294377129189715, + "tokens_seen": 49020928 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4233260154724121, + "objective/train/avg_token_score": 0.01209855917841196, + "objective/train/docs_used": 39958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.3168957233428955, + "objective/train/original_loss": 5.3540496826171875, + "objective/train/theoretical_loss": 5.292220566937567, + "objective/train/tokens_used": 49152000, + "objective/train/unlikelihood_loss": 0.07595264166593552, + "theoretical_loss": 5.292220566937567, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975325977933802, + "loss": 2.3305, + "theoretical_loss": 5.292220566937567, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975125376128386, + "loss": 2.3771, + "theoretical_loss": 5.290071353216895, + "tokens_seen": 49283072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974924774322969, + "loss": 2.1448, + "theoretical_loss": 5.28792944354311, + "tokens_seen": 49414144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974724172517553, + "loss": 2.3501, + "theoretical_loss": 5.285794793817999, + "tokens_seen": 49545216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974523570712136, + "loss": 2.0919, + "theoretical_loss": 5.283667360325351, + "tokens_seen": 49676288 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497432296890672, + "loss": 2.1672, + "theoretical_loss": 5.281547099726654, + "tokens_seen": 49807360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974122367101304, + "loss": 2.2725, + "theoretical_loss": 5.279433969056848, + "tokens_seen": 49938432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973921765295888, + "loss": 2.3125, + "theoretical_loss": 5.277327925720137, + "tokens_seen": 50069504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973721163490472, + "loss": 2.1681, + "theoretical_loss": 5.275228927485855, + "tokens_seen": 50200576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973520561685056, + "loss": 2.1653, + "theoretical_loss": 5.273136932484399, + "tokens_seen": 50331648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973319959879639, + "loss": 2.4031, + "theoretical_loss": 5.271051899203207, + "tokens_seen": 50462720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973119358074223, + "loss": 2.0327, + "theoretical_loss": 5.268973786482794, + "tokens_seen": 50593792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972918756268806, + "loss": 2.1619, + "theoretical_loss": 5.266902553512847, + "tokens_seen": 50724864 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.25061094760894775, + "objective/train/avg_token_score": 0.02978723682463169, + "objective/train/docs_used": 40584, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3529692888259888, + "objective/train/original_loss": 4.955077648162842, + "objective/train/theoretical_loss": 5.265869504275602, + "objective/train/tokens_used": 50790400, + "objective/train/unlikelihood_loss": 0.16319940984249115, + "theoretical_loss": 5.265869504275602, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497271815446339, + "loss": 2.1136, + "theoretical_loss": 5.264838159828369, + "tokens_seen": 50855936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972517552657974, + "loss": 2.3096, + "theoretical_loss": 5.262780565305875, + "tokens_seen": 50987008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972316950852557, + "loss": 2.3798, + "theoretical_loss": 5.260729730159641, + "tokens_seen": 51118080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972116349047142, + "loss": 2.3046, + "theoretical_loss": 5.2586856149380035, + "tokens_seen": 51249152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971915747241726, + "loss": 2.3706, + "theoretical_loss": 5.256648180519708, + "tokens_seen": 51380224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971715145436309, + "loss": 2.0973, + "theoretical_loss": 5.25461738811031, + "tokens_seen": 51511296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971514543630893, + "loss": 2.6331, + "theoretical_loss": 5.252593199238619, + "tokens_seen": 51642368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313941825477, + "loss": 2.2023, + "theoretical_loss": 5.2505755757531904, + "tokens_seen": 51773440 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497111334002006, + "loss": 2.1155, + "theoretical_loss": 5.248564479818876, + "tokens_seen": 51904512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970912738214644, + "loss": 2.3576, + "theoretical_loss": 5.246559873913396, + "tokens_seen": 52035584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970712136409227, + "loss": 2.0642, + "theoretical_loss": 5.24456172082399, + "tokens_seen": 52166656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970511534603811, + "loss": 2.393, + "theoretical_loss": 5.242569983644074, + "tokens_seen": 52297728 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.42399805784225464, + "objective/train/avg_token_score": 0.009449027478694916, + "objective/train/docs_used": 41941, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1671478748321533, + "objective/train/original_loss": 5.030263423919678, + "objective/train/theoretical_loss": 5.240584625769978, + "objective/train/tokens_used": 52428800, + "objective/train/unlikelihood_loss": 0.07760672271251678, + "theoretical_loss": 5.240584625769978, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970310932798396, + "loss": 2.2038, + "theoretical_loss": 5.240584625769978, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970110330992979, + "loss": 2.1783, + "theoretical_loss": 5.238605610897698, + "tokens_seen": 52559872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969909729187563, + "loss": 2.1103, + "theoretical_loss": 5.2366329030197125, + "tokens_seen": 52690944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969709127382147, + "loss": 2.1368, + "theoretical_loss": 5.2346664664218245, + "tokens_seen": 52822016 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496950852557673, + "loss": 2.2045, + "theoretical_loss": 5.232706265680049, + "tokens_seen": 52953088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969307923771314, + "loss": 1.911, + "theoretical_loss": 5.230752265657554, + "tokens_seen": 53084160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969107321965897, + "loss": 2.103, + "theoretical_loss": 5.228804431501619, + "tokens_seen": 53215232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968906720160481, + "loss": 2.4849, + "theoretical_loss": 5.226862728640651, + "tokens_seen": 53346304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968706118355065, + "loss": 2.005, + "theoretical_loss": 5.2249271227812315, + "tokens_seen": 53477376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968505516549649, + "loss": 2.2629, + "theoretical_loss": 5.222997579905204, + "tokens_seen": 53608448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968304914744233, + "loss": 2.2283, + "theoretical_loss": 5.2210740662667945, + "tokens_seen": 53739520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968104312938817, + "loss": 2.399, + "theoretical_loss": 5.219156548389775, + "tokens_seen": 53870592 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049679037111334, + "loss": 2.3043, + "theoretical_loss": 5.217244993064664, + "tokens_seen": 54001664 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.42821359634399414, + "objective/train/avg_token_score": 0.012333965860307217, + "objective/train/docs_used": 42708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.223114490509033, + "objective/train/original_loss": 5.237239837646484, + "objective/train/theoretical_loss": 5.216291441053366, + "objective/train/tokens_used": 54067200, + "objective/train/unlikelihood_loss": 0.06622930616140366, + "theoretical_loss": 5.216291441053366, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967703109327984, + "loss": 2.5708, + "theoretical_loss": 5.215339367345955, + "tokens_seen": 54132736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967502507522568, + "loss": 2.3715, + "theoretical_loss": 5.2134396385493815, + "tokens_seen": 54263808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967301905717151, + "loss": 2.1358, + "theoretical_loss": 5.211545774249233, + "tokens_seen": 54394880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967101303911735, + "loss": 2.4727, + "theoretical_loss": 5.209657742275683, + "tokens_seen": 54525952 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496690070210632, + "loss": 2.102, + "theoretical_loss": 5.207775510712159, + "tokens_seen": 54657024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966700100300903, + "loss": 2.2568, + "theoretical_loss": 5.205899047892753, + "tokens_seen": 54788096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966499498495487, + "loss": 1.8779, + "theoretical_loss": 5.204028322399658, + "tokens_seen": 54919168 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496629889669007, + "loss": 2.4808, + "theoretical_loss": 5.202163303060633, + "tokens_seen": 55050240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966098294884654, + "loss": 2.1792, + "theoretical_loss": 5.20030395894651, + "tokens_seen": 55181312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965897693079238, + "loss": 2.3935, + "theoretical_loss": 5.198450259368721, + "tokens_seen": 55312384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965697091273821, + "loss": 2.2388, + "theoretical_loss": 5.196602173876867, + "tokens_seen": 55443456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965496489468405, + "loss": 2.209, + "theoretical_loss": 5.194759672256309, + "tokens_seen": 55574528 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.34781280159950256, + "objective/train/avg_token_score": 0.039099015295505524, + "objective/train/docs_used": 44116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9161739349365234, + "objective/train/original_loss": 5.530422687530518, + "objective/train/theoretical_loss": 5.192922724525789, + "objective/train/tokens_used": 55705600, + "objective/train/unlikelihood_loss": 0.06317880749702454, + "theoretical_loss": 5.192922724525789, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965295887662988, + "loss": 2.2517, + "theoretical_loss": 5.192922724525789, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965095285857573, + "loss": 2.2552, + "theoretical_loss": 5.19109130093509, + "tokens_seen": 55836672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964894684052157, + "loss": 2.5353, + "theoretical_loss": 5.189265371962712, + "tokens_seen": 55967744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496469408224674, + "loss": 2.2548, + "theoretical_loss": 5.187444908313586, + "tokens_seen": 56098816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964493480441324, + "loss": 2.3618, + "theoretical_loss": 5.185629880916814, + "tokens_seen": 56229888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964292878635908, + "loss": 1.9889, + "theoretical_loss": 5.18382026092343, + "tokens_seen": 56360960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964092276830491, + "loss": 2.6223, + "theoretical_loss": 5.182016019704204, + "tokens_seen": 56492032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963891675025075, + "loss": 2.2145, + "theoretical_loss": 5.180217128847451, + "tokens_seen": 56623104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963691073219659, + "loss": 2.0782, + "theoretical_loss": 5.178423560156894, + "tokens_seen": 56754176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963490471414242, + "loss": 2.1989, + "theoretical_loss": 5.176635285649521, + "tokens_seen": 56885248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963289869608827, + "loss": 2.174, + "theoretical_loss": 5.174852277553498, + "tokens_seen": 57016320 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496308926780341, + "loss": 2.2559, + "theoretical_loss": 5.17307450830609, + "tokens_seen": 57147392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962888665997994, + "loss": 2.221, + "theoretical_loss": 5.1713019505516105, + "tokens_seen": 57278464 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.39821603894233704, + "objective/train/avg_token_score": 0.009783880785107613, + "objective/train/docs_used": 44698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.974114179611206, + "objective/train/original_loss": 5.170925617218018, + "objective/train/theoretical_loss": 5.170417617491682, + "objective/train/tokens_used": 57344000, + "objective/train/unlikelihood_loss": 0.07565154135227203, + "theoretical_loss": 5.170417617491682, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962688064192578, + "loss": 2.1205, + "theoretical_loss": 5.169534577139395, + "tokens_seen": 57409536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962487462387161, + "loss": 1.8749, + "theoretical_loss": 5.167772361121805, + "tokens_seen": 57540608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962286860581746, + "loss": 2.1519, + "theoretical_loss": 5.1660152757522475, + "tokens_seen": 57671680 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496208625877633, + "loss": 2.2882, + "theoretical_loss": 5.164263294483226, + "tokens_seen": 57802752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961885656970913, + "loss": 2.175, + "theoretical_loss": 5.162516390964408, + "tokens_seen": 57933824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961685055165497, + "loss": 1.834, + "theoretical_loss": 5.160774539040716, + "tokens_seen": 58064896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961484453360081, + "loss": 2.0703, + "theoretical_loss": 5.159037712750455, + "tokens_seen": 58195968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961283851554664, + "loss": 2.0747, + "theoretical_loss": 5.157305886323435, + "tokens_seen": 58327040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961083249749248, + "loss": 2.2992, + "theoretical_loss": 5.155579034179144, + "tokens_seen": 58458112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960882647943831, + "loss": 2.192, + "theoretical_loss": 5.153857130924929, + "tokens_seen": 58589184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960682046138415, + "loss": 2.4796, + "theoretical_loss": 5.152140151354191, + "tokens_seen": 58720256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960481444333, + "loss": 2.053, + "theoretical_loss": 5.150428070444621, + "tokens_seen": 58851328 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.42919111251831055, + "objective/train/avg_token_score": 0.023179521784186363, + "objective/train/docs_used": 46033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9980182647705078, + "objective/train/original_loss": 5.299570560455322, + "objective/train/theoretical_loss": 5.1487208633564405, + "objective/train/tokens_used": 58982400, + "objective/train/unlikelihood_loss": 0.05876965448260307, + "theoretical_loss": 5.1487208633564405, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960280842527583, + "loss": 2.0719, + "theoretical_loss": 5.1487208633564405, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960080240722167, + "loss": 2.1807, + "theoretical_loss": 5.147018505430666, + "tokens_seen": 59113472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959879638916751, + "loss": 1.8661, + "theoretical_loss": 5.145320972187402, + "tokens_seen": 59244544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959679037111334, + "loss": 2.313, + "theoretical_loss": 5.143628239324139, + "tokens_seen": 59375616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959478435305918, + "loss": 2.3328, + "theoretical_loss": 5.1419402827140885, + "tokens_seen": 59506688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959277833500501, + "loss": 2.192, + "theoretical_loss": 5.140257078404524, + "tokens_seen": 59637760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959077231695085, + "loss": 2.0898, + "theoretical_loss": 5.138578602615146, + "tokens_seen": 59768832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958876629889669, + "loss": 2.1709, + "theoretical_loss": 5.1369048317364685, + "tokens_seen": 59899904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958676028084253, + "loss": 2.0217, + "theoretical_loss": 5.135235742328217, + "tokens_seen": 60030976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958475426278837, + "loss": 2.1712, + "theoretical_loss": 5.133571311117755, + "tokens_seen": 60162048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958274824473421, + "loss": 2.1534, + "theoretical_loss": 5.131911514998518, + "tokens_seen": 60293120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958074222668004, + "loss": 2.0424, + "theoretical_loss": 5.130256331028474, + "tokens_seen": 60424192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957873620862588, + "loss": 2.0909, + "theoretical_loss": 5.128605736428597, + "tokens_seen": 60555264 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.31408846378326416, + "objective/train/avg_token_score": 0.03549068421125412, + "objective/train/docs_used": 46630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6117610931396484, + "objective/train/original_loss": 5.108785152435303, + "objective/train/theoretical_loss": 5.127782153068225, + "objective/train/tokens_used": 60620800, + "objective/train/unlikelihood_loss": 0.07724561542272568, + "theoretical_loss": 5.127782153068225, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957673019057172, + "loss": 2.0965, + "theoretical_loss": 5.126959708581356, + "tokens_seen": 60686336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957472417251755, + "loss": 2.104, + "theoretical_loss": 5.125318225029231, + "tokens_seen": 60817408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957271815446339, + "loss": 2.107, + "theoretical_loss": 5.12368126347323, + "tokens_seen": 60948480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957071213640923, + "loss": 2.6384, + "theoretical_loss": 5.122048801771443, + "tokens_seen": 61079552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956870611835507, + "loss": 2.4012, + "theoretical_loss": 5.120420817937591, + "tokens_seen": 61210624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956670010030091, + "loss": 2.2013, + "theoretical_loss": 5.118797290139605, + "tokens_seen": 61341696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956469408224674, + "loss": 2.0302, + "theoretical_loss": 5.1171781966982195, + "tokens_seen": 61472768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956268806419258, + "loss": 2.0847, + "theoretical_loss": 5.11556351608558, + "tokens_seen": 61603840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956068204613842, + "loss": 2.2934, + "theoretical_loss": 5.113953226923864, + "tokens_seen": 61734912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955867602808425, + "loss": 2.5557, + "theoretical_loss": 5.112347307983919, + "tokens_seen": 61865984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955667001003009, + "loss": 2.1684, + "theoretical_loss": 5.110745738183919, + "tokens_seen": 61997056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955466399197592, + "loss": 2.2634, + "theoretical_loss": 5.109148496588032, + "tokens_seen": 62128128 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.44507575035095215, + "objective/train/avg_token_score": 0.011363647878170013, + "objective/train/docs_used": 47896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.073281764984131, + "objective/train/original_loss": 4.909066200256348, + "objective/train/theoretical_loss": 5.107555562405102, + "objective/train/tokens_used": 62259200, + "objective/train/unlikelihood_loss": 0.08025174587965012, + "theoretical_loss": 5.107555562405102, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955265797392177, + "loss": 2.0933, + "theoretical_loss": 5.107555562405102, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955065195586761, + "loss": 2.0567, + "theoretical_loss": 5.105966914987349, + "tokens_seen": 62390272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954864593781344, + "loss": 2.1778, + "theoretical_loss": 5.10438253382908, + "tokens_seen": 62521344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954663991975928, + "loss": 2.406, + "theoretical_loss": 5.102802398565418, + "tokens_seen": 62652416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954463390170512, + "loss": 2.2442, + "theoretical_loss": 5.101226488971042, + "tokens_seen": 62783488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954262788365095, + "loss": 1.9896, + "theoretical_loss": 5.09965478495894, + "tokens_seen": 62914560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954062186559679, + "loss": 1.906, + "theoretical_loss": 5.09808726657918, + "tokens_seen": 63045632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953861584754263, + "loss": 2.3592, + "theoretical_loss": 5.096523914017688, + "tokens_seen": 63176704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953660982948846, + "loss": 2.2084, + "theoretical_loss": 5.094964707595047, + "tokens_seen": 63307776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953460381143431, + "loss": 2.1077, + "theoretical_loss": 5.093409627765306, + "tokens_seen": 63438848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953259779338014, + "loss": 2.1148, + "theoretical_loss": 5.091858655114796, + "tokens_seen": 63569920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953059177532598, + "loss": 2.2873, + "theoretical_loss": 5.090311770360971, + "tokens_seen": 63700992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952858575727182, + "loss": 2.1432, + "theoretical_loss": 5.088768954351249, + "tokens_seen": 63832064 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.5087976455688477, + "objective/train/avg_token_score": 0.013270500116050243, + "objective/train/docs_used": 48551, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.6218936443328857, + "objective/train/original_loss": 5.463879108428955, + "objective/train/theoretical_loss": 5.087999066175502, + "objective/train/tokens_used": 63897600, + "objective/train/unlikelihood_loss": 0.0694500133395195, + "theoretical_loss": 5.087999066175502, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952657973921765, + "loss": 2.1419, + "theoretical_loss": 5.0872301880618735, + "tokens_seen": 63963136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952457372116349, + "loss": 1.9254, + "theoretical_loss": 5.085695452596788, + "tokens_seen": 64094208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952256770310933, + "loss": 1.9951, + "theoretical_loss": 5.0841647291865115, + "tokens_seen": 64225280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952056168505516, + "loss": 2.2865, + "theoretical_loss": 5.082637999187046, + "tokens_seen": 64356352 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049518555667001, + "loss": 2.0893, + "theoretical_loss": 5.0811152440787755, + "tokens_seen": 64487424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951654964894685, + "loss": 2.2674, + "theoretical_loss": 5.079596445465386, + "tokens_seen": 64618496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951454363089268, + "loss": 2.0007, + "theoretical_loss": 5.078081585072802, + "tokens_seen": 64749568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951253761283852, + "loss": 2.0058, + "theoretical_loss": 5.076570644748123, + "tokens_seen": 64880640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951053159478435, + "loss": 1.895, + "theoretical_loss": 5.075063606458576, + "tokens_seen": 65011712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950852557673019, + "loss": 2.2423, + "theoretical_loss": 5.07356045229049, + "tokens_seen": 65142784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950651955867603, + "loss": 2.0065, + "theoretical_loss": 5.072061164448261, + "tokens_seen": 65273856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950451354062186, + "loss": 2.1188, + "theoretical_loss": 5.070565725253344, + "tokens_seen": 65404928 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.34683528542518616, + "objective/train/avg_token_score": 0.024793358519673347, + "objective/train/docs_used": 49128, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.808795690536499, + "objective/train/original_loss": 5.487620830535889, + "objective/train/theoretical_loss": 5.069074117143246, + "objective/train/tokens_used": 65536000, + "objective/train/unlikelihood_loss": 0.0768253281712532, + "theoretical_loss": 5.069074117143246, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495025075225677, + "loss": 1.8184, + "theoretical_loss": 5.069074117143246, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950050150451354, + "loss": 1.9084, + "theoretical_loss": 5.067586322670541, + "tokens_seen": 65667072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949849548645938, + "loss": 2.3579, + "theoretical_loss": 5.066102324501883, + "tokens_seen": 65798144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949648946840522, + "loss": 2.1562, + "theoretical_loss": 5.064622105417033, + "tokens_seen": 65929216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949448345035105, + "loss": 2.0672, + "theoretical_loss": 5.063145648307904, + "tokens_seen": 66060288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949247743229689, + "loss": 1.9244, + "theoretical_loss": 5.061672936177604, + "tokens_seen": 66191360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949047141424273, + "loss": 1.8588, + "theoretical_loss": 5.060203952139497, + "tokens_seen": 66322432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948846539618856, + "loss": 1.9714, + "theoretical_loss": 5.0587386794162725, + "tokens_seen": 66453504 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494864593781344, + "loss": 2.1933, + "theoretical_loss": 5.057277101339023, + "tokens_seen": 66584576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948445336008024, + "loss": 1.9907, + "theoretical_loss": 5.055819201346331, + "tokens_seen": 66715648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948244734202607, + "loss": 2.2394, + "theoretical_loss": 5.054364962983367, + "tokens_seen": 66846720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948044132397192, + "loss": 2.1289, + "theoretical_loss": 5.052914369900997, + "tokens_seen": 66977792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947843530591776, + "loss": 2.362, + "theoretical_loss": 5.051467405854897, + "tokens_seen": 67108864 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.3599095940589905, + "objective/train/avg_token_score": 0.01841607131063938, + "objective/train/docs_used": 50658, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9209699630737305, + "objective/train/original_loss": 5.435420036315918, + "objective/train/theoretical_loss": 5.05074527967254, + "objective/train/tokens_used": 67174400, + "objective/train/unlikelihood_loss": 0.08842965960502625, + "theoretical_loss": 5.05074527967254, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947642928786359, + "loss": 2.086, + "theoretical_loss": 5.050024054704677, + "tokens_seen": 67239936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947442326980943, + "loss": 1.9711, + "theoretical_loss": 5.048584300413019, + "tokens_seen": 67371008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947241725175527, + "loss": 2.2735, + "theoretical_loss": 5.0471481270448155, + "tokens_seen": 67502080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947041123370111, + "loss": 2.0652, + "theoretical_loss": 5.045715518766322, + "tokens_seen": 67633152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946840521564695, + "loss": 1.9284, + "theoretical_loss": 5.044286459844319, + "tokens_seen": 67764224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946639919759278, + "loss": 2.1492, + "theoretical_loss": 5.0428609346452795, + "tokens_seen": 67895296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946439317953862, + "loss": 2.3224, + "theoretical_loss": 5.041438927634549, + "tokens_seen": 68026368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946238716148446, + "loss": 2.1179, + "theoretical_loss": 5.040020423375525, + "tokens_seen": 68157440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946038114343029, + "loss": 2.1086, + "theoretical_loss": 5.038605406528857, + "tokens_seen": 68288512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945837512537613, + "loss": 2.042, + "theoretical_loss": 5.037193861851646, + "tokens_seen": 68419584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945636910732196, + "loss": 2.3074, + "theoretical_loss": 5.035785774196654, + "tokens_seen": 68550656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945436308926781, + "loss": 2.2202, + "theoretical_loss": 5.034381128511525, + "tokens_seen": 68681728 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.3374877870082855, + "objective/train/avg_token_score": 0.023196162655949593, + "objective/train/docs_used": 51428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4241688251495361, + "objective/train/original_loss": 4.484879493713379, + "objective/train/theoretical_loss": 5.032979909838007, + "objective/train/tokens_used": 68812800, + "objective/train/unlikelihood_loss": 0.11043024063110352, + "theoretical_loss": 5.032979909838007, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945235707121365, + "loss": 1.958, + "theoretical_loss": 5.032979909838007, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945035105315948, + "loss": 2.023, + "theoretical_loss": 5.031582103311187, + "tokens_seen": 68943872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944834503510532, + "loss": 1.984, + "theoretical_loss": 5.030187694158739, + "tokens_seen": 69074944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944633901705116, + "loss": 2.2235, + "theoretical_loss": 5.028796667700159, + "tokens_seen": 69206016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944433299899699, + "loss": 2.0468, + "theoretical_loss": 5.0274090093460355, + "tokens_seen": 69337088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944232698094283, + "loss": 2.2206, + "theoretical_loss": 5.0260247045973045, + "tokens_seen": 69468160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944032096288867, + "loss": 2.1891, + "theoretical_loss": 5.024643739044526, + "tokens_seen": 69599232 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494383149448345, + "loss": 2.2133, + "theoretical_loss": 5.023266098367161, + "tokens_seen": 69730304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943630892678035, + "loss": 2.1522, + "theoretical_loss": 5.0218917683328534, + "tokens_seen": 69861376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943430290872618, + "loss": 1.9963, + "theoretical_loss": 5.020520734796728, + "tokens_seen": 69992448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943229689067202, + "loss": 2.0446, + "theoretical_loss": 5.019152983700687, + "tokens_seen": 70123520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943029087261786, + "loss": 2.1678, + "theoretical_loss": 5.01778850107272, + "tokens_seen": 70254592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828485456369, + "loss": 2.3371, + "theoretical_loss": 5.016427273026212, + "tokens_seen": 70385664 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4471529722213745, + "objective/train/avg_token_score": 0.013587483204901218, + "objective/train/docs_used": 52769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.070784568786621, + "objective/train/original_loss": 5.132371425628662, + "objective/train/theoretical_loss": 5.015747875154622, + "objective/train/tokens_used": 70451200, + "objective/train/unlikelihood_loss": 0.07507549971342087, + "theoretical_loss": 5.015747875154622, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942627883650953, + "loss": 1.8887, + "theoretical_loss": 5.015069285759269, + "tokens_seen": 70516736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942427281845537, + "loss": 2.0888, + "theoretical_loss": 5.0137145255540405, + "tokens_seen": 70647808 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494222668004012, + "loss": 2.0747, + "theoretical_loss": 5.012362978776057, + "tokens_seen": 70778880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942026078234704, + "loss": 1.994, + "theoretical_loss": 5.011014631873566, + "tokens_seen": 70909952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941825476429289, + "loss": 2.2445, + "theoretical_loss": 5.009669471376882, + "tokens_seen": 71041024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941624874623872, + "loss": 2.2045, + "theoretical_loss": 5.008327483897736, + "tokens_seen": 71172096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941424272818456, + "loss": 2.0795, + "theoretical_loss": 5.006988656128635, + "tokens_seen": 71303168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941223671013039, + "loss": 2.2727, + "theoretical_loss": 5.00565297484223, + "tokens_seen": 71434240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941023069207623, + "loss": 2.0863, + "theoretical_loss": 5.004320426890686, + "tokens_seen": 71565312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940822467402207, + "loss": 2.2777, + "theoretical_loss": 5.002990999205057, + "tokens_seen": 71696384 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494062186559679, + "loss": 2.0031, + "theoretical_loss": 5.001664678794671, + "tokens_seen": 71827456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940421263791374, + "loss": 2.0639, + "theoretical_loss": 5.0003414527465235, + "tokens_seen": 71958528 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4258919954299927, + "objective/train/avg_token_score": 0.020259426906704903, + "objective/train/docs_used": 53229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.168090343475342, + "objective/train/original_loss": 5.140798091888428, + "objective/train/theoretical_loss": 4.999021308224664, + "objective/train/tokens_used": 72089600, + "objective/train/unlikelihood_loss": 0.11368393152952194, + "theoretical_loss": 4.999021308224664, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940220661985958, + "loss": 2.0618, + "theoretical_loss": 4.999021308224664, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940020060180542, + "loss": 1.9749, + "theoretical_loss": 4.997704232469606, + "tokens_seen": 72220672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939819458375126, + "loss": 2.1991, + "theoretical_loss": 4.996390212797728, + "tokens_seen": 72351744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493961885656971, + "loss": 2.288, + "theoretical_loss": 4.995079236600686, + "tokens_seen": 72482816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939418254764293, + "loss": 1.9311, + "theoretical_loss": 4.993771291344839, + "tokens_seen": 72613888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939217652958877, + "loss": 1.9251, + "theoretical_loss": 4.992466364570659, + "tokens_seen": 72744960 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493901705115346, + "loss": 1.8568, + "theoretical_loss": 4.991164443892175, + "tokens_seen": 72876032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938816449348044, + "loss": 2.2996, + "theoretical_loss": 4.989865516996396, + "tokens_seen": 73007104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938615847542628, + "loss": 2.1793, + "theoretical_loss": 4.988569571642756, + "tokens_seen": 73138176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938415245737211, + "loss": 2.0888, + "theoretical_loss": 4.9872765956625615, + "tokens_seen": 73269248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938214643931796, + "loss": 2.0231, + "theoretical_loss": 4.9859865769584335, + "tokens_seen": 73400320 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493801404212638, + "loss": 2.1265, + "theoretical_loss": 4.984699503503771, + "tokens_seen": 73531392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937813440320963, + "loss": 2.1259, + "theoretical_loss": 4.9834153633422105, + "tokens_seen": 73662464 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.43444526195526123, + "objective/train/avg_token_score": 0.0062065767124295235, + "objective/train/docs_used": 53865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.2150537967681885, + "objective/train/original_loss": 5.1299591064453125, + "objective/train/theoretical_loss": 4.982774389529053, + "objective/train/tokens_used": 73728000, + "objective/train/unlikelihood_loss": 0.08083882927894592, + "theoretical_loss": 4.982774389529053, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937612838515547, + "loss": 1.9495, + "theoretical_loss": 4.9821341445870875, + "tokens_seen": 73793536 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493741223671013, + "loss": 1.9215, + "theoretical_loss": 4.98085583542091, + "tokens_seen": 73924608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937211634904714, + "loss": 2.021, + "theoretical_loss": 4.979580424094836, + "tokens_seen": 74055680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937011033099298, + "loss": 1.8973, + "theoretical_loss": 4.978307898928149, + "tokens_seen": 74186752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936810431293881, + "loss": 2.1481, + "theoretical_loss": 4.97703824830775, + "tokens_seen": 74317824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936609829488465, + "loss": 1.8232, + "theoretical_loss": 4.975771460687641, + "tokens_seen": 74448896 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493640922768305, + "loss": 1.8557, + "theoretical_loss": 4.974507524588424, + "tokens_seen": 74579968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936208625877633, + "loss": 1.8265, + "theoretical_loss": 4.973246428596802, + "tokens_seen": 74711040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936008024072217, + "loss": 2.192, + "theoretical_loss": 4.971988161365077, + "tokens_seen": 74842112 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049358074222668, + "loss": 1.9407, + "theoretical_loss": 4.970732711610667, + "tokens_seen": 74973184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935606820461384, + "loss": 2.2415, + "theoretical_loss": 4.96948006811561, + "tokens_seen": 75104256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935406218655968, + "loss": 2.0764, + "theoretical_loss": 4.968230219726093, + "tokens_seen": 75235328 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.41452834010124207, + "objective/train/avg_token_score": 0.014995137229561806, + "objective/train/docs_used": 54881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9440230131149292, + "objective/train/original_loss": 4.934903144836426, + "objective/train/theoretical_loss": 4.966983155351962, + "objective/train/tokens_used": 75366400, + "objective/train/unlikelihood_loss": 0.07033995538949966, + "theoretical_loss": 4.966983155351962, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935205616850551, + "loss": 2.1972, + "theoretical_loss": 4.966983155351962, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935005015045135, + "loss": 2.145, + "theoretical_loss": 4.96573886396626, + "tokens_seen": 75497472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934804413239719, + "loss": 1.9567, + "theoretical_loss": 4.964497334604748, + "tokens_seen": 75628544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934603811434303, + "loss": 1.9129, + "theoretical_loss": 4.963258556365449, + "tokens_seen": 75759616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934403209628887, + "loss": 2.0695, + "theoretical_loss": 4.962022518408183, + "tokens_seen": 75890688 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493420260782347, + "loss": 2.2744, + "theoretical_loss": 4.9607892099541075, + "tokens_seen": 76021760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934002006018054, + "loss": 2.2838, + "theoretical_loss": 4.959558620285274, + "tokens_seen": 76152832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933801404212638, + "loss": 1.9697, + "theoretical_loss": 4.958330738744172, + "tokens_seen": 76283904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933600802407221, + "loss": 2.2717, + "theoretical_loss": 4.957105554733289, + "tokens_seen": 76414976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933400200601805, + "loss": 2.0457, + "theoretical_loss": 4.955883057714669, + "tokens_seen": 76546048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933199598796389, + "loss": 2.0651, + "theoretical_loss": 4.954663237209477, + "tokens_seen": 76677120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932998996990972, + "loss": 2.0885, + "theoretical_loss": 4.9534460827975675, + "tokens_seen": 76808192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932798395185557, + "loss": 2.2084, + "theoretical_loss": 4.952231584117056, + "tokens_seen": 76939264 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.3369990289211273, + "objective/train/avg_token_score": 0.013007127679884434, + "objective/train/docs_used": 55424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6116856336593628, + "objective/train/original_loss": 4.758896350860596, + "objective/train/theoretical_loss": 4.951625327453812, + "objective/train/tokens_used": 77004800, + "objective/train/unlikelihood_loss": 0.08679111301898956, + "theoretical_loss": 4.951625327453812, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932597793380141, + "loss": 1.8865, + "theoretical_loss": 4.951019730863894, + "tokens_seen": 77070336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932397191574724, + "loss": 2.1724, + "theoretical_loss": 4.94981051279145, + "tokens_seen": 77201408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932196589769308, + "loss": 2.1451, + "theoretical_loss": 4.948603919710088, + "tokens_seen": 77332480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931995987963893, + "loss": 1.9967, + "theoretical_loss": 4.947399941486762, + "tokens_seen": 77463552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931795386158476, + "loss": 2.005, + "theoretical_loss": 4.946198568044602, + "tokens_seen": 77594624 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493159478435306, + "loss": 2.2193, + "theoretical_loss": 4.944999789362508, + "tokens_seen": 77725696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931394182547643, + "loss": 1.9879, + "theoretical_loss": 4.9438035954747495, + "tokens_seen": 77856768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931193580742227, + "loss": 2.0917, + "theoretical_loss": 4.942609976470566, + "tokens_seen": 77987840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930992978936811, + "loss": 2.0912, + "theoretical_loss": 4.941418922493774, + "tokens_seen": 78118912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930792377131394, + "loss": 2.1054, + "theoretical_loss": 4.940230423742372, + "tokens_seen": 78249984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930591775325978, + "loss": 2.302, + "theoretical_loss": 4.939044470468156, + "tokens_seen": 78381056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930391173520562, + "loss": 2.1726, + "theoretical_loss": 4.937861052976332, + "tokens_seen": 78512128 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4615102708339691, + "objective/train/avg_token_score": 0.007493212353438139, + "objective/train/docs_used": 56825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.250938892364502, + "objective/train/original_loss": 4.587989330291748, + "objective/train/theoretical_loss": 4.9366801616251355, + "objective/train/tokens_used": 78643200, + "objective/train/unlikelihood_loss": 0.1308807134628296, + "theoretical_loss": 4.9366801616251355, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930190571715146, + "loss": 2.0669, + "theoretical_loss": 4.9366801616251355, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492998996990973, + "loss": 1.8013, + "theoretical_loss": 4.935501786825457, + "tokens_seen": 78774272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929789368104313, + "loss": 1.9622, + "theoretical_loss": 4.934325919040461, + "tokens_seen": 78905344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929588766298897, + "loss": 1.9327, + "theoretical_loss": 4.933152548785222, + "tokens_seen": 79036416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929388164493481, + "loss": 2.1009, + "theoretical_loss": 4.931981666626351, + "tokens_seen": 79167488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929187562688064, + "loss": 2.2199, + "theoretical_loss": 4.930813263181631, + "tokens_seen": 79298560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928986960882648, + "loss": 1.9262, + "theoretical_loss": 4.929647329119659, + "tokens_seen": 79429632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928786359077232, + "loss": 2.1295, + "theoretical_loss": 4.928483855159485, + "tokens_seen": 79560704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928585757271815, + "loss": 2.0885, + "theoretical_loss": 4.9273228320702565, + "tokens_seen": 79691776 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049283851554664, + "loss": 2.2262, + "theoretical_loss": 4.926164250670868, + "tokens_seen": 79822848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928184553660984, + "loss": 2.1799, + "theoretical_loss": 4.925008101829608, + "tokens_seen": 79953920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927983951855567, + "loss": 2.0411, + "theoretical_loss": 4.923854376463816, + "tokens_seen": 80084992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927783350050151, + "loss": 2.0662, + "theoretical_loss": 4.92270306553954, + "tokens_seen": 80216064 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.46413734555244446, + "objective/train/avg_token_score": 0.03281039744615555, + "objective/train/docs_used": 57296, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.3605399131774902, + "objective/train/original_loss": 5.119383811950684, + "objective/train/theoretical_loss": 4.922128312683096, + "objective/train/tokens_used": 80281600, + "objective/train/unlikelihood_loss": 0.08986959606409073, + "theoretical_loss": 4.922128312683096, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927582748244734, + "loss": 1.9299, + "theoretical_loss": 4.921554160071194, + "tokens_seen": 80347136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927382146439318, + "loss": 2.0743, + "theoretical_loss": 4.920407651121222, + "tokens_seen": 80478208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927181544633902, + "loss": 2.0166, + "theoretical_loss": 4.919263529799759, + "tokens_seen": 80609280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926980942828485, + "loss": 1.9821, + "theoretical_loss": 4.91812178726431, + "tokens_seen": 80740352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926780341023069, + "loss": 1.962, + "theoretical_loss": 4.916982414719408, + "tokens_seen": 80871424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926579739217654, + "loss": 1.8903, + "theoretical_loss": 4.915845403416299, + "tokens_seen": 81002496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926379137412237, + "loss": 2.0539, + "theoretical_loss": 4.914710744652614, + "tokens_seen": 81133568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926178535606821, + "loss": 2.0817, + "theoretical_loss": 4.913578429772047, + "tokens_seen": 81264640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925977933801404, + "loss": 2.0058, + "theoretical_loss": 4.912448450164041, + "tokens_seen": 81395712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925777331995988, + "loss": 1.741, + "theoretical_loss": 4.911320797263471, + "tokens_seen": 81526784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925576730190572, + "loss": 2.1737, + "theoretical_loss": 4.910195462550334, + "tokens_seen": 81657856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925376128385155, + "loss": 2.246, + "theoretical_loss": 4.909072437549434, + "tokens_seen": 81788928 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.46144917607307434, + "objective/train/avg_token_score": 0.00516983587294817, + "objective/train/docs_used": 58605, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.2914438247680664, + "objective/train/original_loss": 4.962143421173096, + "objective/train/theoretical_loss": 4.907951713830082, + "objective/train/tokens_used": 81920000, + "objective/train/unlikelihood_loss": 0.07921816408634186, + "theoretical_loss": 4.907951713830082, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925175526579739, + "loss": 2.07, + "theoretical_loss": 4.907951713830082, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924974924774323, + "loss": 2.0655, + "theoretical_loss": 4.906833283005785, + "tokens_seen": 82051072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924774322968907, + "loss": 2.093, + "theoretical_loss": 4.90571713673395, + "tokens_seen": 82182144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924573721163491, + "loss": 2.3312, + "theoretical_loss": 4.904603266715578, + "tokens_seen": 82313216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924373119358075, + "loss": 2.0983, + "theoretical_loss": 4.903491664694977, + "tokens_seen": 82444288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924172517552658, + "loss": 1.9503, + "theoretical_loss": 4.902382322459456, + "tokens_seen": 82575360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923971915747242, + "loss": 2.1464, + "theoretical_loss": 4.90127523183904, + "tokens_seen": 82706432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923771313941825, + "loss": 2.1856, + "theoretical_loss": 4.900170384706181, + "tokens_seen": 82837504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923570712136409, + "loss": 1.9381, + "theoretical_loss": 4.899067772975469, + "tokens_seen": 82968576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923370110330993, + "loss": 2.3138, + "theoretical_loss": 4.897967388603346, + "tokens_seen": 83099648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923169508525576, + "loss": 2.0845, + "theoretical_loss": 4.896869223587828, + "tokens_seen": 83230720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922968906720161, + "loss": 2.1112, + "theoretical_loss": 4.895773269968219, + "tokens_seen": 83361792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922768304914745, + "loss": 1.9807, + "theoretical_loss": 4.894679519824841, + "tokens_seen": 83492864 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4049975574016571, + "objective/train/avg_token_score": 0.012781642377376556, + "objective/train/docs_used": 59310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9555177688598633, + "objective/train/original_loss": 5.0520477294921875, + "objective/train/theoretical_loss": 4.894133468592984, + "objective/train/tokens_used": 83558400, + "objective/train/unlikelihood_loss": 0.09169325232505798, + "theoretical_loss": 4.894133468592984, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922567703109328, + "loss": 1.7533, + "theoretical_loss": 4.8935879652787495, + "tokens_seen": 83623936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922367101303912, + "loss": 1.9519, + "theoretical_loss": 4.892498598491473, + "tokens_seen": 83755008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922166499498495, + "loss": 2.1345, + "theoretical_loss": 4.891411411664727, + "tokens_seen": 83886080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921965897693079, + "loss": 2.1, + "theoretical_loss": 4.890326397040158, + "tokens_seen": 84017152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921765295887663, + "loss": 2.0142, + "theoretical_loss": 4.8892435468990705, + "tokens_seen": 84148224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921564694082246, + "loss": 2.0119, + "theoretical_loss": 4.888162853562166, + "tokens_seen": 84279296 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492136409227683, + "loss": 2.2767, + "theoretical_loss": 4.88708430938928, + "tokens_seen": 84410368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921163490471415, + "loss": 2.1413, + "theoretical_loss": 4.8860079067791204, + "tokens_seen": 84541440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920962888665998, + "loss": 2.0281, + "theoretical_loss": 4.884933638169014, + "tokens_seen": 84672512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920762286860582, + "loss": 1.802, + "theoretical_loss": 4.883861496034644, + "tokens_seen": 84803584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920561685055166, + "loss": 1.8336, + "theoretical_loss": 4.8827914728898065, + "tokens_seen": 84934656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920361083249749, + "loss": 2.0471, + "theoretical_loss": 4.881723561286149, + "tokens_seen": 85065728 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.35856547951698303, + "objective/train/avg_token_score": 0.008686823770403862, + "objective/train/docs_used": 60492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7955634593963623, + "objective/train/original_loss": 4.565763473510742, + "objective/train/theoretical_loss": 4.880657753812926, + "objective/train/tokens_used": 85196800, + "objective/train/unlikelihood_loss": 0.19698937237262726, + "theoretical_loss": 4.880657753812926, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920160481444333, + "loss": 1.9216, + "theoretical_loss": 4.880657753812926, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919959879638916, + "loss": 2.1886, + "theoretical_loss": 4.879594043096755, + "tokens_seen": 85327872 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049197592778335, + "loss": 2.0444, + "theoretical_loss": 4.87853242180136, + "tokens_seen": 85458944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919558676028084, + "loss": 1.9928, + "theoretical_loss": 4.877472882627343, + "tokens_seen": 85590016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919358074222668, + "loss": 1.9912, + "theoretical_loss": 4.876415418311928, + "tokens_seen": 85721088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919157472417252, + "loss": 1.9497, + "theoretical_loss": 4.875360021628733, + "tokens_seen": 85852160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918956870611836, + "loss": 2.1111, + "theoretical_loss": 4.874306685387525, + "tokens_seen": 85983232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918756268806419, + "loss": 2.2713, + "theoretical_loss": 4.87325540243399, + "tokens_seen": 86114304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918555667001003, + "loss": 2.0874, + "theoretical_loss": 4.872206165649493, + "tokens_seen": 86245376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918355065195586, + "loss": 1.9654, + "theoretical_loss": 4.871158967950852, + "tokens_seen": 86376448 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491815446339017, + "loss": 2.0332, + "theoretical_loss": 4.8701138022901045, + "tokens_seen": 86507520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917953861584754, + "loss": 2.2051, + "theoretical_loss": 4.8690706616542805, + "tokens_seen": 86638592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917753259779337, + "loss": 2.0213, + "theoretical_loss": 4.868029539065176, + "tokens_seen": 86769664 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.45521751046180725, + "objective/train/avg_token_score": 0.008304348215460777, + "objective/train/docs_used": 61005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1370649337768555, + "objective/train/original_loss": 4.633736610412598, + "objective/train/theoretical_loss": 4.867509732366907, + "objective/train/tokens_used": 86835200, + "objective/train/unlikelihood_loss": 0.09587273001670837, + "theoretical_loss": 4.867509732366907, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917552657973922, + "loss": 1.7873, + "theoretical_loss": 4.866990427579129, + "tokens_seen": 86900736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917352056168506, + "loss": 1.9276, + "theoretical_loss": 4.8659533202867955, + "tokens_seen": 87031808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917151454363089, + "loss": 1.7595, + "theoretical_loss": 4.864918210312927, + "tokens_seen": 87162880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916950852557673, + "loss": 2.029, + "theoretical_loss": 4.863885090816158, + "tokens_seen": 87293952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916750250752258, + "loss": 1.8672, + "theoretical_loss": 4.862853954988781, + "tokens_seen": 87425024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916549648946841, + "loss": 1.9145, + "theoretical_loss": 4.861824796056533, + "tokens_seen": 87556096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916349047141425, + "loss": 1.8864, + "theoretical_loss": 4.860797607278385, + "tokens_seen": 87687168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916148445336008, + "loss": 2.0223, + "theoretical_loss": 4.859772381946323, + "tokens_seen": 87818240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915947843530592, + "loss": 1.8535, + "theoretical_loss": 4.858749113385144, + "tokens_seen": 87949312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915747241725176, + "loss": 1.8087, + "theoretical_loss": 4.8577277949522415, + "tokens_seen": 88080384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915546639919759, + "loss": 1.8308, + "theoretical_loss": 4.856708420037402, + "tokens_seen": 88211456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915346038114343, + "loss": 1.8766, + "theoretical_loss": 4.8556909820625975, + "tokens_seen": 88342528 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.44208210706710815, + "objective/train/avg_token_score": 0.02312779426574707, + "objective/train/docs_used": 61597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.139789581298828, + "objective/train/original_loss": 4.875903129577637, + "objective/train/theoretical_loss": 4.854675474481779, + "objective/train/tokens_used": 88473600, + "objective/train/unlikelihood_loss": 0.07984444499015808, + "theoretical_loss": 4.854675474481779, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915145436308927, + "loss": 2.214, + "theoretical_loss": 4.854675474481779, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914944834503511, + "loss": 1.9395, + "theoretical_loss": 4.85366189078068, + "tokens_seen": 88604672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914744232698095, + "loss": 1.9757, + "theoretical_loss": 4.852650224476609, + "tokens_seen": 88735744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914543630892679, + "loss": 2.0743, + "theoretical_loss": 4.851640469118255, + "tokens_seen": 88866816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343029087262, + "loss": 2.041, + "theoretical_loss": 4.850632618285486, + "tokens_seen": 88997888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914142427281846, + "loss": 2.0281, + "theoretical_loss": 4.849626665589156, + "tokens_seen": 89128960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913941825476429, + "loss": 1.8351, + "theoretical_loss": 4.8486226046709024, + "tokens_seen": 89260032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913741223671013, + "loss": 2.1069, + "theoretical_loss": 4.847620429202967, + "tokens_seen": 89391104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913540621865597, + "loss": 1.7571, + "theoretical_loss": 4.846620132887992, + "tokens_seen": 89522176 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491334002006018, + "loss": 2.0421, + "theoretical_loss": 4.845621709458831, + "tokens_seen": 89653248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913139418254765, + "loss": 1.859, + "theoretical_loss": 4.844625152678364, + "tokens_seen": 89784320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912938816449349, + "loss": 2.0505, + "theoretical_loss": 4.84363045633931, + "tokens_seen": 89915392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912738214643932, + "loss": 1.7393, + "theoretical_loss": 4.84263761426404, + "tokens_seen": 90046464 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.34017595648765564, + "objective/train/avg_token_score": 0.00968579389154911, + "objective/train/docs_used": 63098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.632453441619873, + "objective/train/original_loss": 4.724969387054443, + "objective/train/theoretical_loss": 4.842141886652876, + "objective/train/tokens_used": 90112000, + "objective/train/unlikelihood_loss": 0.09992973506450653, + "theoretical_loss": 4.842141886652876, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912537612838516, + "loss": 2.0554, + "theoretical_loss": 4.841646620304388, + "tokens_seen": 90177536 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049123370110331, + "loss": 2.1008, + "theoretical_loss": 4.840657468341476, + "tokens_seen": 90308608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912136409227683, + "loss": 1.9921, + "theoretical_loss": 4.839670152285526, + "tokens_seen": 90439680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911935807422267, + "loss": 2.072, + "theoretical_loss": 4.838684666075682, + "tokens_seen": 90570752 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491173520561685, + "loss": 1.9095, + "theoretical_loss": 4.837701003679829, + "tokens_seen": 90701824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911534603811434, + "loss": 1.877, + "theoretical_loss": 4.836719159094422, + "tokens_seen": 90832896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911334002006019, + "loss": 1.9558, + "theoretical_loss": 4.835739126344298, + "tokens_seen": 90963968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911133400200602, + "loss": 1.9039, + "theoretical_loss": 4.834760899482514, + "tokens_seen": 91095040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910932798395186, + "loss": 2.1059, + "theoretical_loss": 4.833784472590165, + "tokens_seen": 91226112 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491073219658977, + "loss": 1.9337, + "theoretical_loss": 4.832809839776213, + "tokens_seen": 91357184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910531594784353, + "loss": 2.2944, + "theoretical_loss": 4.831836995177319, + "tokens_seen": 91488256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910330992978937, + "loss": 1.9808, + "theoretical_loss": 4.8308659329576695, + "tokens_seen": 91619328 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.19037145376205444, + "objective/train/avg_token_score": 0.031226102262735367, + "objective/train/docs_used": 63840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9143338203430176, + "objective/train/original_loss": 4.837706089019775, + "objective/train/theoretical_loss": 4.8298966473088125, + "objective/train/tokens_used": 91750400, + "objective/train/unlikelihood_loss": 0.17378956079483032, + "theoretical_loss": 4.8298966473088125, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491013039117352, + "loss": 1.7083, + "theoretical_loss": 4.8298966473088125, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909929789368104, + "loss": 1.9804, + "theoretical_loss": 4.8289291324494865, + "tokens_seen": 91881472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909729187562688, + "loss": 1.9059, + "theoretical_loss": 4.827963382625454, + "tokens_seen": 92012544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909528585757272, + "loss": 1.9218, + "theoretical_loss": 4.826999392109344, + "tokens_seen": 92143616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909327983951856, + "loss": 1.8156, + "theoretical_loss": 4.826037155200478, + "tokens_seen": 92274688 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490912738214644, + "loss": 1.9604, + "theoretical_loss": 4.825076666224717, + "tokens_seen": 92405760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908926780341023, + "loss": 1.8492, + "theoretical_loss": 4.824117919534297, + "tokens_seen": 92536832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908726178535607, + "loss": 2.0128, + "theoretical_loss": 4.823160909507665, + "tokens_seen": 92667904 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490852557673019, + "loss": 1.7759, + "theoretical_loss": 4.822205630549329, + "tokens_seen": 92798976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908324974924774, + "loss": 1.9094, + "theoretical_loss": 4.821252077089696, + "tokens_seen": 92930048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908124373119358, + "loss": 1.9877, + "theoretical_loss": 4.820300243584913, + "tokens_seen": 93061120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907923771313941, + "loss": 1.8244, + "theoretical_loss": 4.819350124516717, + "tokens_seen": 93192192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907723169508526, + "loss": 1.9063, + "theoretical_loss": 4.818401714392279, + "tokens_seen": 93323264 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.49071359634399414, + "objective/train/avg_token_score": 0.014383784495294094, + "objective/train/docs_used": 65224, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1994926929473877, + "objective/train/original_loss": 4.691728115081787, + "objective/train/theoretical_loss": 4.817928148474378, + "objective/train/tokens_used": 93388800, + "objective/train/unlikelihood_loss": 0.06607113778591156, + "theoretical_loss": 4.817928148474378, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490752256770311, + "loss": 1.9917, + "theoretical_loss": 4.817455007744052, + "tokens_seen": 93454336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907321965897693, + "loss": 2.0181, + "theoretical_loss": 4.816509999129618, + "tokens_seen": 93585408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907121364092277, + "loss": 1.9882, + "theoretical_loss": 4.815566683131536, + "tokens_seen": 93716480 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490692076228686, + "loss": 1.9407, + "theoretical_loss": 4.814625054357199, + "tokens_seen": 93847552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906720160481444, + "loss": 1.7596, + "theoretical_loss": 4.813685107438679, + "tokens_seen": 93978624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906519558676028, + "loss": 1.8142, + "theoretical_loss": 4.812746837032582, + "tokens_seen": 94109696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906318956870611, + "loss": 1.6129, + "theoretical_loss": 4.811810237819904, + "tokens_seen": 94240768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906118355065195, + "loss": 1.9627, + "theoretical_loss": 4.810875304505881, + "tokens_seen": 94371840 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490591775325978, + "loss": 1.9738, + "theoretical_loss": 4.809942031819853, + "tokens_seen": 94502912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905717151454363, + "loss": 1.7409, + "theoretical_loss": 4.809010414515113, + "tokens_seen": 94633984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905516549648947, + "loss": 2.0122, + "theoretical_loss": 4.808080447368766, + "tokens_seen": 94765056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905315947843531, + "loss": 1.8492, + "theoretical_loss": 4.807152125181597, + "tokens_seen": 94896128 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.5782013535499573, + "objective/train/avg_token_score": 0.0024078439455479383, + "objective/train/docs_used": 65494, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.5631401538848877, + "objective/train/original_loss": 4.700851917266846, + "objective/train/theoretical_loss": 4.8062254427779205, + "objective/train/tokens_used": 95027200, + "objective/train/unlikelihood_loss": 0.061158839613199234, + "theoretical_loss": 4.8062254427779205, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905115346038114, + "loss": 1.9437, + "theoretical_loss": 4.8062254427779205, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904914744232698, + "loss": 1.9594, + "theoretical_loss": 4.805300395005444, + "tokens_seen": 95158272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904714142427281, + "loss": 1.9803, + "theoretical_loss": 4.8043769767351385, + "tokens_seen": 95289344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904513540621865, + "loss": 1.9562, + "theoretical_loss": 4.803455182861087, + "tokens_seen": 95420416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904312938816449, + "loss": 2.0377, + "theoretical_loss": 4.802535008300364, + "tokens_seen": 95551488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904112337011033, + "loss": 2.0507, + "theoretical_loss": 4.801616447992888, + "tokens_seen": 95682560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903911735205617, + "loss": 2.073, + "theoretical_loss": 4.8006994969013, + "tokens_seen": 95813632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903711133400201, + "loss": 2.0056, + "theoretical_loss": 4.799784150010819, + "tokens_seen": 95944704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903510531594784, + "loss": 1.9311, + "theoretical_loss": 4.798870402329115, + "tokens_seen": 96075776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903309929789368, + "loss": 1.7706, + "theoretical_loss": 4.797958248886179, + "tokens_seen": 96206848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903109327983952, + "loss": 1.8998, + "theoretical_loss": 4.797047684734192, + "tokens_seen": 96337920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902908726178535, + "loss": 1.9212, + "theoretical_loss": 4.796138704947397, + "tokens_seen": 96468992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902708124373119, + "loss": 1.6952, + "theoretical_loss": 4.795231304621968, + "tokens_seen": 96600064 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.32086998224258423, + "objective/train/avg_token_score": 0.05917775630950928, + "objective/train/docs_used": 66055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3923687934875488, + "objective/train/original_loss": 5.291764259338379, + "objective/train/theoretical_loss": 4.794778195230787, + "objective/train/tokens_used": 96665600, + "objective/train/unlikelihood_loss": 0.06859153509140015, + "theoretical_loss": 4.794778195230787, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902507522567703, + "loss": 1.8366, + "theoretical_loss": 4.794325478875885, + "tokens_seen": 96731136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902306920762287, + "loss": 2.0257, + "theoretical_loss": 4.793421222848808, + "tokens_seen": 96862208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902106318956871, + "loss": 1.9104, + "theoretical_loss": 4.792518531701948, + "tokens_seen": 96993280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901905717151454, + "loss": 1.9913, + "theoretical_loss": 4.791617400617948, + "tokens_seen": 97124352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901705115346038, + "loss": 1.8546, + "theoretical_loss": 4.790717824800755, + "tokens_seen": 97255424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901504513540623, + "loss": 1.8143, + "theoretical_loss": 4.789819799475499, + "tokens_seen": 97386496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901303911735206, + "loss": 1.8901, + "theoretical_loss": 4.788923319888369, + "tokens_seen": 97517568 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490110330992979, + "loss": 1.8487, + "theoretical_loss": 4.788028381306497, + "tokens_seen": 97648640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900902708124374, + "loss": 1.868, + "theoretical_loss": 4.787134979017832, + "tokens_seen": 97779712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900702106318957, + "loss": 1.8523, + "theoretical_loss": 4.786243108331024, + "tokens_seen": 97910784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900501504513541, + "loss": 1.7257, + "theoretical_loss": 4.785352764575304, + "tokens_seen": 98041856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900300902708124, + "loss": 1.9306, + "theoretical_loss": 4.784463943100367, + "tokens_seen": 98172928 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4745234549045563, + "objective/train/avg_token_score": 0.020105870440602303, + "objective/train/docs_used": 66835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.3115997314453125, + "objective/train/original_loss": 4.947286128997803, + "objective/train/theoretical_loss": 4.783576639276257, + "objective/train/tokens_used": 98304000, + "objective/train/unlikelihood_loss": 0.07566744834184647, + "theoretical_loss": 4.783576639276257, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900100300902708, + "loss": 1.7746, + "theoretical_loss": 4.783576639276257, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899899699097292, + "loss": 1.8378, + "theoretical_loss": 4.782690848493245, + "tokens_seen": 98435072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899699097291876, + "loss": 2.0137, + "theoretical_loss": 4.781806566161723, + "tokens_seen": 98566144 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489949849548646, + "loss": 2.0675, + "theoretical_loss": 4.78092378771208, + "tokens_seen": 98697216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899297893681044, + "loss": 2.0216, + "theoretical_loss": 4.780042508594596, + "tokens_seen": 98828288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899097291875627, + "loss": 1.7411, + "theoretical_loss": 4.779162724279324, + "tokens_seen": 98959360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898896690070211, + "loss": 1.7954, + "theoretical_loss": 4.778284430255981, + "tokens_seen": 99090432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898696088264794, + "loss": 1.9314, + "theoretical_loss": 4.777407622033838, + "tokens_seen": 99221504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898495486459378, + "loss": 1.8941, + "theoretical_loss": 4.776532295141601, + "tokens_seen": 99352576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898294884653962, + "loss": 2.0467, + "theoretical_loss": 4.775658445127318, + "tokens_seen": 99483648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898094282848545, + "loss": 1.875, + "theoretical_loss": 4.7747860675582485, + "tokens_seen": 99614720 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489789368104313, + "loss": 1.8207, + "theoretical_loss": 4.773915158020776, + "tokens_seen": 99745792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897693079237714, + "loss": 1.8993, + "theoretical_loss": 4.773045712120284, + "tokens_seen": 99876864 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.3229472041130066, + "objective/train/avg_token_score": 0.02979409135878086, + "objective/train/docs_used": 67525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.47669517993927, + "objective/train/original_loss": 4.8227763175964355, + "objective/train/theoretical_loss": 4.77261153666576, + "objective/train/tokens_used": 99942400, + "objective/train/unlikelihood_loss": 0.1576315313577652, + "theoretical_loss": 4.77261153666576, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897492477432297, + "loss": 1.9601, + "theoretical_loss": 4.772177725481062, + "tokens_seen": 100007936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897291875626881, + "loss": 1.7757, + "theoretical_loss": 4.771311193746191, + "tokens_seen": 100139008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897091273821465, + "loss": 1.8233, + "theoretical_loss": 4.770446112577445, + "tokens_seen": 100270080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896890672016048, + "loss": 1.9513, + "theoretical_loss": 4.769582477655177, + "tokens_seen": 100401152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896690070210632, + "loss": 1.8272, + "theoretical_loss": 4.768720284678228, + "tokens_seen": 100532224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896489468405215, + "loss": 1.764, + "theoretical_loss": 4.767859529363809, + "tokens_seen": 100663296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896288866599799, + "loss": 1.956, + "theoretical_loss": 4.767000207447417, + "tokens_seen": 100794368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896088264794384, + "loss": 1.9563, + "theoretical_loss": 4.766142314682716, + "tokens_seen": 100925440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895887662988967, + "loss": 1.8045, + "theoretical_loss": 4.765285846841444, + "tokens_seen": 101056512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895687061183551, + "loss": 2.0327, + "theoretical_loss": 4.764430799713314, + "tokens_seen": 101187584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895486459378135, + "loss": 1.6355, + "theoretical_loss": 4.763577169105912, + "tokens_seen": 101318656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895285857572718, + "loss": 1.9126, + "theoretical_loss": 4.762724950844598, + "tokens_seen": 101449728 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.361131489276886, + "objective/train/avg_token_score": 0.010854127816855907, + "objective/train/docs_used": 68617, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5994964838027954, + "objective/train/original_loss": 4.556946754455566, + "objective/train/theoretical_loss": 4.761874140772408, + "objective/train/tokens_used": 101580800, + "objective/train/unlikelihood_loss": 0.12181789427995682, + "theoretical_loss": 4.761874140772408, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895085255767302, + "loss": 2.0572, + "theoretical_loss": 4.761874140772408, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894884653961885, + "loss": 1.6931, + "theoretical_loss": 4.761024734749958, + "tokens_seen": 101711872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894684052156469, + "loss": 1.842, + "theoretical_loss": 4.760176728655345, + "tokens_seen": 101842944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894483450351053, + "loss": 1.7938, + "theoretical_loss": 4.759330118384053, + "tokens_seen": 101974016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894282848545637, + "loss": 1.8328, + "theoretical_loss": 4.758484899848854, + "tokens_seen": 102105088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894082246740221, + "loss": 1.8125, + "theoretical_loss": 4.7576410689797175, + "tokens_seen": 102236160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893881644934805, + "loss": 1.6074, + "theoretical_loss": 4.756798621723712, + "tokens_seen": 102367232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893681043129388, + "loss": 1.7143, + "theoretical_loss": 4.755957554044917, + "tokens_seen": 102498304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893480441323972, + "loss": 2.0152, + "theoretical_loss": 4.755117861924321, + "tokens_seen": 102629376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893279839518556, + "loss": 1.8091, + "theoretical_loss": 4.754279541359738, + "tokens_seen": 102760448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893079237713139, + "loss": 1.8848, + "theoretical_loss": 4.75344258836571, + "tokens_seen": 102891520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892878635907723, + "loss": 1.9612, + "theoretical_loss": 4.752606998973421, + "tokens_seen": 103022592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892678034102307, + "loss": 1.7873, + "theoretical_loss": 4.7517727692306035, + "tokens_seen": 103153664 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.3702346086502075, + "objective/train/avg_token_score": 0.0168976578861475, + "objective/train/docs_used": 69265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7375133037567139, + "objective/train/original_loss": 4.848740100860596, + "objective/train/theoretical_loss": 4.751356162997164, + "objective/train/tokens_used": 103219200, + "objective/train/unlikelihood_loss": 0.1075160950422287, + "theoretical_loss": 4.751356162997164, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892477432296891, + "loss": 2.047, + "theoretical_loss": 4.750939895201443, + "tokens_seen": 103284736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892276830491475, + "loss": 1.7528, + "theoretical_loss": 4.750108372966501, + "tokens_seen": 103415808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892076228686058, + "loss": 1.9577, + "theoretical_loss": 4.749278198622617, + "tokens_seen": 103546880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891875626880642, + "loss": 1.7456, + "theoretical_loss": 4.748449368282822, + "tokens_seen": 103677952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891675025075226, + "loss": 1.6757, + "theoretical_loss": 4.747621878076252, + "tokens_seen": 103809024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891474423269809, + "loss": 1.6139, + "theoretical_loss": 4.746795724148061, + "tokens_seen": 103940096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891273821464393, + "loss": 2.0421, + "theoretical_loss": 4.745970902659338, + "tokens_seen": 104071168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891073219658976, + "loss": 1.6816, + "theoretical_loss": 4.7451474097870125, + "tokens_seen": 104202240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890872617853561, + "loss": 1.763, + "theoretical_loss": 4.744325241723777, + "tokens_seen": 104333312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890672016048145, + "loss": 1.7075, + "theoretical_loss": 4.743504394678, + "tokens_seen": 104464384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890471414242728, + "loss": 1.8281, + "theoretical_loss": 4.742684864873641, + "tokens_seen": 104595456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890270812437312, + "loss": 1.7182, + "theoretical_loss": 4.741866648550168, + "tokens_seen": 104726528 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.35599949955940247, + "objective/train/avg_token_score": 0.05716551095247269, + "objective/train/docs_used": 70556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.608228087425232, + "objective/train/original_loss": 4.970082759857178, + "objective/train/theoretical_loss": 4.741049741962473, + "objective/train/tokens_used": 104857600, + "objective/train/unlikelihood_loss": 0.09267803281545639, + "theoretical_loss": 4.741049741962473, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890070210631896, + "loss": 1.6529, + "theoretical_loss": 4.741049741962473, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889869608826479, + "loss": 1.8281, + "theoretical_loss": 4.740234141380794, + "tokens_seen": 104988672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889669007021063, + "loss": 1.7417, + "theoretical_loss": 4.739419843090626, + "tokens_seen": 105119744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889468405215647, + "loss": 1.7041, + "theoretical_loss": 4.738606843392644, + "tokens_seen": 105250816 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488926780341023, + "loss": 1.5751, + "theoretical_loss": 4.737795138602624, + "tokens_seen": 105381888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889067201604815, + "loss": 1.8483, + "theoretical_loss": 4.736984725051357, + "tokens_seen": 105512960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888866599799398, + "loss": 1.7904, + "theoretical_loss": 4.736175599084576, + "tokens_seen": 105644032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888665997993982, + "loss": 1.9943, + "theoretical_loss": 4.735367757062869, + "tokens_seen": 105775104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888465396188566, + "loss": 1.7043, + "theoretical_loss": 4.734561195361609, + "tokens_seen": 105906176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888264794383149, + "loss": 1.9254, + "theoretical_loss": 4.733755910370867, + "tokens_seen": 106037248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888064192577733, + "loss": 1.7947, + "theoretical_loss": 4.732951898495341, + "tokens_seen": 106168320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887863590772317, + "loss": 1.7915, + "theoretical_loss": 4.732149156154276, + "tokens_seen": 106299392 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048876629889669, + "loss": 1.7682, + "theoretical_loss": 4.731347679781386, + "tokens_seen": 106430464 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.429068922996521, + "objective/train/avg_token_score": 0.009868655353784561, + "objective/train/docs_used": 70950, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9191758632659912, + "objective/train/original_loss": 4.865683078765869, + "objective/train/theoretical_loss": 4.730947415222481, + "objective/train/tokens_used": 106496000, + "objective/train/unlikelihood_loss": 0.10545031726360321, + "theoretical_loss": 4.730947415222481, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887462387161484, + "loss": 1.8063, + "theoretical_loss": 4.730547465824781, + "tokens_seen": 106561536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887261785356069, + "loss": 2.0496, + "theoretical_loss": 4.72974851074689, + "tokens_seen": 106692608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887061183550652, + "loss": 1.869, + "theoretical_loss": 4.728950811024383, + "tokens_seen": 106823680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886860581745236, + "loss": 1.82, + "theoretical_loss": 4.728154363148102, + "tokens_seen": 106954752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886659979939819, + "loss": 1.8344, + "theoretical_loss": 4.72735916362298, + "tokens_seen": 107085824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886459378134403, + "loss": 1.6656, + "theoretical_loss": 4.726565208967973, + "tokens_seen": 107216896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886258776328988, + "loss": 1.9221, + "theoretical_loss": 4.725772495715983, + "tokens_seen": 107347968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886058174523571, + "loss": 1.8658, + "theoretical_loss": 4.724981020413787, + "tokens_seen": 107479040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885857572718155, + "loss": 1.8208, + "theoretical_loss": 4.7241907796219635, + "tokens_seen": 107610112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656970912739, + "loss": 1.748, + "theoretical_loss": 4.723401769914824, + "tokens_seen": 107741184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885456369107322, + "loss": 1.9757, + "theoretical_loss": 4.722613987880335, + "tokens_seen": 107872256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885255767301906, + "loss": 1.7293, + "theoretical_loss": 4.721827430120053, + "tokens_seen": 108003328 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4198435842990875, + "objective/train/avg_token_score": 0.03244104981422424, + "objective/train/docs_used": 72030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9712913036346436, + "objective/train/original_loss": 5.18431282043457, + "objective/train/theoretical_loss": 4.721042093249051, + "objective/train/tokens_used": 108134400, + "objective/train/unlikelihood_loss": 0.07222175598144531, + "theoretical_loss": 4.721042093249051, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488505516549649, + "loss": 2.0079, + "theoretical_loss": 4.721042093249051, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884854563691073, + "loss": 1.8136, + "theoretical_loss": 4.7202579738958494, + "tokens_seen": 108265472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884653961885657, + "loss": 1.6593, + "theoretical_loss": 4.719475068702346, + "tokens_seen": 108396544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884453360080241, + "loss": 1.8464, + "theoretical_loss": 4.718693374323747, + "tokens_seen": 108527616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884252758274825, + "loss": 1.552, + "theoretical_loss": 4.717912887428501, + "tokens_seen": 108658688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884052156469409, + "loss": 1.7877, + "theoretical_loss": 4.717133604698222, + "tokens_seen": 108789760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883851554663992, + "loss": 1.6993, + "theoretical_loss": 4.716355522827633, + "tokens_seen": 108920832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883650952858576, + "loss": 1.87, + "theoretical_loss": 4.715578638524491, + "tokens_seen": 109051904 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488345035105316, + "loss": 1.7024, + "theoretical_loss": 4.714802948509522, + "tokens_seen": 109182976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883249749247743, + "loss": 1.7157, + "theoretical_loss": 4.714028449516356, + "tokens_seen": 109314048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883049147442327, + "loss": 1.8335, + "theoretical_loss": 4.713255138291454, + "tokens_seen": 109445120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882848545636911, + "loss": 1.7999, + "theoretical_loss": 4.712483011594056, + "tokens_seen": 109576192 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048826479438314946, + "loss": 1.6147, + "theoretical_loss": 4.7117120661961005, + "tokens_seen": 109707264 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.3784213066101074, + "objective/train/avg_token_score": 0.07327920943498611, + "objective/train/docs_used": 72698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.72416353225708, + "objective/train/original_loss": 4.901858806610107, + "objective/train/theoretical_loss": 4.711327035479103, + "objective/train/tokens_used": 109772800, + "objective/train/unlikelihood_loss": 0.09020828455686569, + "theoretical_loss": 4.711327035479103, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882447342026078, + "loss": 1.836, + "theoretical_loss": 4.710942298882169, + "tokens_seen": 109838336 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822467402206624, + "loss": 1.8356, + "theoretical_loss": 4.710173706449419, + "tokens_seen": 109969408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882046138415246, + "loss": 1.6668, + "theoretical_loss": 4.7094062857075185, + "tokens_seen": 110100480 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818455366098296, + "loss": 1.843, + "theoretical_loss": 4.708640033478584, + "tokens_seen": 110231552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881644934804413, + "loss": 1.6348, + "theoretical_loss": 4.7078749465971175, + "tokens_seen": 110362624 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048814443329989974, + "loss": 1.885, + "theoretical_loss": 4.707111021909941, + "tokens_seen": 110493696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881243731193581, + "loss": 1.4851, + "theoretical_loss": 4.706348256276138, + "tokens_seen": 110624768 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048810431293881646, + "loss": 1.9024, + "theoretical_loss": 4.705586646566987, + "tokens_seen": 110755840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880842527582748, + "loss": 1.8789, + "theoretical_loss": 4.704826189665905, + "tokens_seen": 110886912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880641925777332, + "loss": 1.9667, + "theoretical_loss": 4.7040668824683785, + "tokens_seen": 111017984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880441323971916, + "loss": 1.7731, + "theoretical_loss": 4.7033087218819105, + "tokens_seen": 111149056 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802407221664997, + "loss": 1.782, + "theoretical_loss": 4.702551704825957, + "tokens_seen": 111280128 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.38880741596221924, + "objective/train/avg_token_score": 0.002843292895704508, + "objective/train/docs_used": 74080, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.625144124031067, + "objective/train/original_loss": 4.229738235473633, + "objective/train/theoretical_loss": 4.701795828231866, + "objective/train/tokens_used": 111411200, + "objective/train/unlikelihood_loss": 0.14396952092647552, + "theoretical_loss": 4.701795828231866, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048800401203610833, + "loss": 1.9266, + "theoretical_loss": 4.701795828231866, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879839518555667, + "loss": 1.5747, + "theoretical_loss": 4.701041089042813, + "tokens_seen": 111542272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879638916750251, + "loss": 1.7214, + "theoretical_loss": 4.700287484213753, + "tokens_seen": 111673344 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048794383149448347, + "loss": 1.9717, + "theoretical_loss": 4.6995350107113545, + "tokens_seen": 111804416 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048792377131394183, + "loss": 1.7515, + "theoretical_loss": 4.698783665513934, + "tokens_seen": 111935488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879037111334002, + "loss": 1.8924, + "theoretical_loss": 4.698033445611415, + "tokens_seen": 112066560 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048788365095285856, + "loss": 1.9578, + "theoretical_loss": 4.697284348005253, + "tokens_seen": 112197632 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048786359077231697, + "loss": 1.5521, + "theoretical_loss": 4.696536369708386, + "tokens_seen": 112328704 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048784353059177533, + "loss": 1.7932, + "theoretical_loss": 4.695789507745176, + "tokens_seen": 112459776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878234704112337, + "loss": 1.8788, + "theoretical_loss": 4.695043759151353, + "tokens_seen": 112590848 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048780341023069206, + "loss": 1.851, + "theoretical_loss": 4.694299120973957, + "tokens_seen": 112721920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877833500501505, + "loss": 1.775, + "theoretical_loss": 4.693555590271282, + "tokens_seen": 112852992 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048776328986960884, + "loss": 1.7527, + "theoretical_loss": 4.692813164112819, + "tokens_seen": 112984064 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.32972875237464905, + "objective/train/avg_token_score": 0.019512729719281197, + "objective/train/docs_used": 74817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5578219890594482, + "objective/train/original_loss": 4.993590354919434, + "objective/train/theoretical_loss": 4.692442364324931, + "objective/train/tokens_used": 113049600, + "objective/train/unlikelihood_loss": 0.07117574661970139, + "theoretical_loss": 4.692442364324931, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877432296890672, + "loss": 1.6282, + "theoretical_loss": 4.692071839579201, + "tokens_seen": 113115136 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048772316950852556, + "loss": 1.7212, + "theoretical_loss": 4.691331613762153, + "tokens_seen": 113246208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877031093279839, + "loss": 1.9666, + "theoretical_loss": 4.690592483764427, + "tokens_seen": 113377280 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048768304914744234, + "loss": 1.8759, + "theoretical_loss": 4.689854446699757, + "tokens_seen": 113508352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876629889669007, + "loss": 1.9312, + "theoretical_loss": 4.689117499692798, + "tokens_seen": 113639424 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048764292878635907, + "loss": 1.9425, + "theoretical_loss": 4.688381639879076, + "tokens_seen": 113770496 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048762286860581743, + "loss": 1.6478, + "theoretical_loss": 4.687646864404934, + "tokens_seen": 113901568 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048760280842527584, + "loss": 1.9203, + "theoretical_loss": 4.686913170427477, + "tokens_seen": 114032640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875827482447342, + "loss": 1.5842, + "theoretical_loss": 4.68618055511452, + "tokens_seen": 114163712 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048756268806419257, + "loss": 1.7842, + "theoretical_loss": 4.685449015644537, + "tokens_seen": 114294784 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048754262788365093, + "loss": 1.6122, + "theoretical_loss": 4.684718549206607, + "tokens_seen": 114425856 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048752256770310935, + "loss": 1.6103, + "theoretical_loss": 4.6839891530003595, + "tokens_seen": 114556928 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.42418134212493896, + "objective/train/avg_token_score": 0.013869252055883408, + "objective/train/docs_used": 76169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.927834153175354, + "objective/train/original_loss": 4.830661296844482, + "objective/train/theoretical_loss": 4.68326082423593, + "objective/train/tokens_used": 114688000, + "objective/train/unlikelihood_loss": 0.13057447969913483, + "theoretical_loss": 4.68326082423593, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875025075225677, + "loss": 1.8594, + "theoretical_loss": 4.68326082423593, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048748244734202607, + "loss": 1.8205, + "theoretical_loss": 4.682533560133901, + "tokens_seen": 114819072 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048746238716148443, + "loss": 1.8782, + "theoretical_loss": 4.681807357925257, + "tokens_seen": 114950144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874423269809428, + "loss": 1.8902, + "theoretical_loss": 4.68108221485133, + "tokens_seen": 115081216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874222668004012, + "loss": 1.7744, + "theoretical_loss": 4.680358128163747, + "tokens_seen": 115212288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874022066198596, + "loss": 1.7876, + "theoretical_loss": 4.6796350951243895, + "tokens_seen": 115343360 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048738214643931794, + "loss": 1.6889, + "theoretical_loss": 4.678913113005333, + "tokens_seen": 115474432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873620862587763, + "loss": 2.0774, + "theoretical_loss": 4.678192179088802, + "tokens_seen": 115605504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873420260782347, + "loss": 1.8764, + "theoretical_loss": 4.677472290667122, + "tokens_seen": 115736576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873219658976931, + "loss": 1.8666, + "theoretical_loss": 4.676753445042669, + "tokens_seen": 115867648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048730190571715144, + "loss": 1.7627, + "theoretical_loss": 4.6760356395278215, + "tokens_seen": 115998720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872818455366098, + "loss": 1.9188, + "theoretical_loss": 4.675318871444908, + "tokens_seen": 116129792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048726178535606816, + "loss": 1.9219, + "theoretical_loss": 4.674603138126168, + "tokens_seen": 116260864 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.44238758087158203, + "objective/train/avg_token_score": 0.016831381246447563, + "objective/train/docs_used": 76931, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.068297863006592, + "objective/train/original_loss": 4.9380340576171875, + "objective/train/theoretical_loss": 4.674245658672382, + "objective/train/tokens_used": 116326400, + "objective/train/unlikelihood_loss": 0.09188218414783478, + "theoretical_loss": 4.674245658672382, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872417251755266, + "loss": 1.8908, + "theoretical_loss": 4.673888436913694, + "tokens_seen": 116391936 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487221664994985, + "loss": 1.5917, + "theoretical_loss": 4.673174765159393, + "tokens_seen": 116523008 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048720160481444336, + "loss": 1.7106, + "theoretical_loss": 4.6724621202249335, + "tokens_seen": 116654080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871815446339017, + "loss": 1.7435, + "theoretical_loss": 4.6717504994817, + "tokens_seen": 116785152 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048716148445336014, + "loss": 2.0931, + "theoretical_loss": 4.671039900310747, + "tokens_seen": 116916224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871414242728185, + "loss": 1.889, + "theoretical_loss": 4.670330320102753, + "tokens_seen": 117047296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048712136409227686, + "loss": 1.8596, + "theoretical_loss": 4.669621756257971, + "tokens_seen": 117178368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871013039117352, + "loss": 1.8924, + "theoretical_loss": 4.668914206186189, + "tokens_seen": 117309440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870812437311936, + "loss": 1.848, + "theoretical_loss": 4.66820766730668, + "tokens_seen": 117440512 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487061183550652, + "loss": 1.6911, + "theoretical_loss": 4.667502137048155, + "tokens_seen": 117571584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048704112337011037, + "loss": 1.7743, + "theoretical_loss": 4.666797612848723, + "tokens_seen": 117702656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048702106318956873, + "loss": 1.6988, + "theoretical_loss": 4.666094092155843, + "tokens_seen": 117833728 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.5015273690223694, + "objective/train/avg_token_score": 0.006345341447740793, + "objective/train/docs_used": 78354, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.065946578979492, + "objective/train/original_loss": 3.913723945617676, + "objective/train/theoretical_loss": 4.665391572426282, + "objective/train/tokens_used": 117964800, + "objective/train/unlikelihood_loss": 0.22304238379001617, + "theoretical_loss": 4.665391572426282, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870010030090271, + "loss": 1.9779, + "theoretical_loss": 4.665391572426282, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869809428284855, + "loss": 1.8665, + "theoretical_loss": 4.664690051126065, + "tokens_seen": 118095872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048696088264794387, + "loss": 1.5626, + "theoretical_loss": 4.66398952573044, + "tokens_seen": 118226944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048694082246740223, + "loss": 1.8862, + "theoretical_loss": 4.663289993723826, + "tokens_seen": 118358016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869207622868606, + "loss": 1.88, + "theoretical_loss": 4.662591452599774, + "tokens_seen": 118489088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048690070210631896, + "loss": 1.8471, + "theoretical_loss": 4.661893899860923, + "tokens_seen": 118620160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688064192577737, + "loss": 1.5922, + "theoretical_loss": 4.661197333018957, + "tokens_seen": 118751232 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686058174523573, + "loss": 1.8316, + "theoretical_loss": 4.6605017495945615, + "tokens_seen": 118882304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868405215646941, + "loss": 1.7042, + "theoretical_loss": 4.659807147117382, + "tokens_seen": 119013376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682046138415246, + "loss": 1.6569, + "theoretical_loss": 4.659113523125981, + "tokens_seen": 119144448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868004012036109, + "loss": 1.6006, + "theoretical_loss": 4.658420875167799, + "tokens_seen": 119275520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678034102306924, + "loss": 1.6682, + "theoretical_loss": 4.657729200799105, + "tokens_seen": 119406592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867602808425276, + "loss": 1.7433, + "theoretical_loss": 4.657038497584967, + "tokens_seen": 119537664 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.359115332365036, + "objective/train/avg_token_score": 0.027856703847646713, + "objective/train/docs_used": 78905, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.605509638786316, + "objective/train/original_loss": 4.950808048248291, + "objective/train/theoretical_loss": 4.656693509402331, + "objective/train/tokens_used": 119603200, + "objective/train/unlikelihood_loss": 0.09441498667001724, + "theoretical_loss": 4.656693509402331, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048674022066198596, + "loss": 1.599, + "theoretical_loss": 4.6563487630991975, + "tokens_seen": 119668736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867201604814443, + "loss": 1.8791, + "theoretical_loss": 4.655659994924323, + "tokens_seen": 119799808 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048670010030090274, + "loss": 1.6709, + "theoretical_loss": 4.65497219065154, + "tokens_seen": 119930880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866800401203611, + "loss": 1.7202, + "theoretical_loss": 4.654285347880672, + "tokens_seen": 120061952 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048665997993981947, + "loss": 1.8856, + "theoretical_loss": 4.653599464220129, + "tokens_seen": 120193024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048663991975927783, + "loss": 1.8096, + "theoretical_loss": 4.6529145372868745, + "tokens_seen": 120324096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048661985957873624, + "loss": 1.6635, + "theoretical_loss": 4.652230564706377, + "tokens_seen": 120455168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865997993981946, + "loss": 1.6887, + "theoretical_loss": 4.651547544112575, + "tokens_seen": 120586240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048657973921765297, + "loss": 1.6575, + "theoretical_loss": 4.650865473147837, + "tokens_seen": 120717312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048655967903711133, + "loss": 1.7481, + "theoretical_loss": 4.650184349462922, + "tokens_seen": 120848384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048653961885656975, + "loss": 1.7805, + "theoretical_loss": 4.6495041707169396, + "tokens_seen": 120979456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865195586760281, + "loss": 1.5922, + "theoretical_loss": 4.648824934577313, + "tokens_seen": 121110528 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.28109726309776306, + "objective/train/avg_token_score": 0.07095440477132797, + "objective/train/docs_used": 79692, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.170123815536499, + "objective/train/original_loss": 4.673448085784912, + "objective/train/theoretical_loss": 4.648146638719739, + "objective/train/tokens_used": 121241600, + "objective/train/unlikelihood_loss": 0.12229027599096298, + "theoretical_loss": 4.648146638719739, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048649949849548647, + "loss": 1.7261, + "theoretical_loss": 4.648146638719739, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048647943831494483, + "loss": 1.8013, + "theoretical_loss": 4.647469280828153, + "tokens_seen": 121372672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864593781344032, + "loss": 1.9773, + "theoretical_loss": 4.646792858594686, + "tokens_seen": 121503744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864393179538616, + "loss": 1.6408, + "theoretical_loss": 4.646117369719629, + "tokens_seen": 121634816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048641925777332, + "loss": 1.7331, + "theoretical_loss": 4.6454428119113995, + "tokens_seen": 121765888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048639919759277834, + "loss": 1.6656, + "theoretical_loss": 4.644769182886495, + "tokens_seen": 121896960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863791374122367, + "loss": 1.8518, + "theoretical_loss": 4.644096480369466, + "tokens_seen": 122028032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863590772316951, + "loss": 1.8672, + "theoretical_loss": 4.6434247020928705, + "tokens_seen": 122159104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863390170511535, + "loss": 1.8686, + "theoretical_loss": 4.642753845797243, + "tokens_seen": 122290176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048631895687061184, + "loss": 1.9569, + "theoretical_loss": 4.642083909231053, + "tokens_seen": 122421248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862988966900702, + "loss": 1.659, + "theoretical_loss": 4.641414890150675, + "tokens_seen": 122552320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048627883650952857, + "loss": 1.8805, + "theoretical_loss": 4.6407467863203475, + "tokens_seen": 122683392 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486258776328987, + "loss": 1.8608, + "theoretical_loss": 4.6400795955121374, + "tokens_seen": 122814464 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.46236559748649597, + "objective/train/avg_token_score": 0.02207406610250473, + "objective/train/docs_used": 81051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.061617851257324, + "objective/train/original_loss": 4.578189849853516, + "objective/train/theoretical_loss": 4.639746341797229, + "objective/train/tokens_used": 122880000, + "objective/train/unlikelihood_loss": 0.10073511302471161, + "theoretical_loss": 4.639746341797229, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048623871614844534, + "loss": 1.9091, + "theoretical_loss": 4.639413315505905, + "tokens_seen": 122945536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862186559679037, + "loss": 1.7783, + "theoretical_loss": 4.638747944089273, + "tokens_seen": 123076608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048619859578736207, + "loss": 1.9328, + "theoretical_loss": 4.638083479057579, + "tokens_seen": 123207680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861785356068205, + "loss": 1.8101, + "theoretical_loss": 4.6374199182138565, + "tokens_seen": 123338752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048615847542627885, + "loss": 1.7789, + "theoretical_loss": 4.636757259368787, + "tokens_seen": 123469824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861384152457372, + "loss": 1.9448, + "theoretical_loss": 4.636095500340669, + "tokens_seen": 123600896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048611835506519557, + "loss": 1.7024, + "theoretical_loss": 4.635434638955388, + "tokens_seen": 123731968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048609829488465393, + "loss": 1.8975, + "theoretical_loss": 4.634774673046376, + "tokens_seen": 123863040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048607823470411235, + "loss": 1.6453, + "theoretical_loss": 4.634115600454582, + "tokens_seen": 123994112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860581745235707, + "loss": 1.8373, + "theoretical_loss": 4.6334574190284314, + "tokens_seen": 124125184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860381143430291, + "loss": 1.7563, + "theoretical_loss": 4.632800126623803, + "tokens_seen": 124256256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048601805416248744, + "loss": 1.832, + "theoretical_loss": 4.632143721103983, + "tokens_seen": 124387328 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.3342497646808624, + "objective/train/avg_token_score": 0.010767764411866665, + "objective/train/docs_used": 81846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.500130295753479, + "objective/train/original_loss": 4.459724426269531, + "objective/train/theoretical_loss": 4.631488200339643, + "objective/train/tokens_used": 124518400, + "objective/train/unlikelihood_loss": 0.17901530861854553, + "theoretical_loss": 4.631488200339643, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048599799398194585, + "loss": 1.8062, + "theoretical_loss": 4.631488200339643, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859779338014042, + "loss": 1.8382, + "theoretical_loss": 4.630833562208797, + "tokens_seen": 124649472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859578736208626, + "loss": 1.7831, + "theoretical_loss": 4.630179804596775, + "tokens_seen": 124780544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048593781344032094, + "loss": 1.7324, + "theoretical_loss": 4.629526925396189, + "tokens_seen": 124911616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859177532597793, + "loss": 1.7767, + "theoretical_loss": 4.628874922506897, + "tokens_seen": 125042688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858976930792377, + "loss": 1.801, + "theoretical_loss": 4.628223793835975, + "tokens_seen": 125173760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858776328986961, + "loss": 1.8394, + "theoretical_loss": 4.627573537297678, + "tokens_seen": 125304832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048585757271815444, + "loss": 1.8139, + "theoretical_loss": 4.6269241508134185, + "tokens_seen": 125435904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858375125376128, + "loss": 1.8361, + "theoretical_loss": 4.6262756323117245, + "tokens_seen": 125566976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858174523570712, + "loss": 2.1512, + "theoretical_loss": 4.625627979728212, + "tokens_seen": 125698048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857973921765296, + "loss": 1.7211, + "theoretical_loss": 4.624981191005554, + "tokens_seen": 125829120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577733199598795, + "loss": 1.7506, + "theoretical_loss": 4.624335264093447, + "tokens_seen": 125960192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857572718154463, + "loss": 1.719, + "theoretical_loss": 4.623690196948582, + "tokens_seen": 126091264 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.2474951148033142, + "objective/train/avg_token_score": 0.018130095675587654, + "objective/train/docs_used": 83055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1577845811843872, + "objective/train/original_loss": 4.456354141235352, + "objective/train/theoretical_loss": 4.623367985152234, + "objective/train/tokens_used": 126156800, + "objective/train/unlikelihood_loss": 0.11982196569442749, + "theoretical_loss": 4.623367985152234, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573721163490467, + "loss": 1.7005, + "theoretical_loss": 4.623045987534609, + "tokens_seen": 126222336 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048571715145436314, + "loss": 2.0355, + "theoretical_loss": 4.622402633822114, + "tokens_seen": 126353408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856970912738215, + "loss": 1.7999, + "theoretical_loss": 4.62176013378858, + "tokens_seen": 126484480 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048567703109327987, + "loss": 1.6439, + "theoretical_loss": 4.621118485418362, + "tokens_seen": 126615552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048565697091273823, + "loss": 2.0235, + "theoretical_loss": 4.620477686702651, + "tokens_seen": 126746624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563691073219664, + "loss": 1.805, + "theoretical_loss": 4.619837735639452, + "tokens_seen": 126877696 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485616850551655, + "loss": 1.8306, + "theoretical_loss": 4.619198630233547, + "tokens_seen": 127008768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559679037111337, + "loss": 1.7948, + "theoretical_loss": 4.618560368496466, + "tokens_seen": 127139840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048557673019057173, + "loss": 1.7666, + "theoretical_loss": 4.617922948446459, + "tokens_seen": 127270912 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048555667001003015, + "loss": 1.8654, + "theoretical_loss": 4.617286368108466, + "tokens_seen": 127401984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855366098294885, + "loss": 1.7467, + "theoretical_loss": 4.616650625514091, + "tokens_seen": 127533056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048551654964894687, + "loss": 1.7312, + "theoretical_loss": 4.616015718701563, + "tokens_seen": 127664128 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.39668866991996765, + "objective/train/avg_token_score": 0.008423499763011932, + "objective/train/docs_used": 83592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7686882019042969, + "objective/train/original_loss": 4.729675769805908, + "objective/train/theoretical_loss": 4.615381645715717, + "objective/train/tokens_used": 127795200, + "objective/train/unlikelihood_loss": 0.08303800970315933, + "theoretical_loss": 4.615381645715717, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048549648946840523, + "loss": 1.8034, + "theoretical_loss": 4.615381645715717, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854764292878636, + "loss": 1.7946, + "theoretical_loss": 4.61474840460796, + "tokens_seen": 127926272 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485456369107322, + "loss": 1.6519, + "theoretical_loss": 4.614115993436242, + "tokens_seen": 128057344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854363089267804, + "loss": 1.9487, + "theoretical_loss": 4.613484410265032, + "tokens_seen": 128188416 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048541624874623874, + "loss": 1.6912, + "theoretical_loss": 4.612853653165283, + "tokens_seen": 128319488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853961885656971, + "loss": 1.7975, + "theoretical_loss": 4.612223720214407, + "tokens_seen": 128450560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853761283851555, + "loss": 1.6292, + "theoretical_loss": 4.61159460949625, + "tokens_seen": 128581632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853560682046139, + "loss": 1.5301, + "theoretical_loss": 4.610966319101056, + "tokens_seen": 128712704 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533600802407224, + "loss": 1.8054, + "theoretical_loss": 4.610338847125445, + "tokens_seen": 128843776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853159478435306, + "loss": 1.6353, + "theoretical_loss": 4.6097121916723856, + "tokens_seen": 128974848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048529588766298897, + "loss": 1.8334, + "theoretical_loss": 4.609086350851165, + "tokens_seen": 129105920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852758274824474, + "loss": 1.7218, + "theoretical_loss": 4.6084613227773605, + "tokens_seen": 129236992 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525576730190574, + "loss": 1.8052, + "theoretical_loss": 4.607837105572816, + "tokens_seen": 129368064 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.5973240733146667, + "objective/train/avg_token_score": 0.010894504375755787, + "objective/train/docs_used": 84814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.5378358364105225, + "objective/train/original_loss": 4.310730934143066, + "objective/train/theoretical_loss": 4.607525300461299, + "objective/train/tokens_used": 129433600, + "objective/train/unlikelihood_loss": 0.1317676305770874, + "theoretical_loss": 4.607525300461299, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852357071213641, + "loss": 1.9714, + "theoretical_loss": 4.607213697365613, + "tokens_seen": 129499136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521564694082247, + "loss": 1.7143, + "theoretical_loss": 4.6065910962900425, + "tokens_seen": 129630208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851955867602809, + "loss": 2.0327, + "theoretical_loss": 4.60596930048658, + "tokens_seen": 129761280 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048517552657973925, + "loss": 1.8787, + "theoretical_loss": 4.6053483081018545, + "tokens_seen": 129892352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851554663991976, + "loss": 1.9663, + "theoretical_loss": 4.604728117288631, + "tokens_seen": 130023424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513540621865597, + "loss": 1.7322, + "theoretical_loss": 4.604108726205774, + "tokens_seen": 130154496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048511534603811433, + "loss": 1.5917, + "theoretical_loss": 4.6034901330182265, + "tokens_seen": 130285568 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048509528585757275, + "loss": 1.7381, + "theoretical_loss": 4.6028723358969845, + "tokens_seen": 130416640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850752256770311, + "loss": 1.8999, + "theoretical_loss": 4.602255333019068, + "tokens_seen": 130547712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850551654964895, + "loss": 1.7749, + "theoretical_loss": 4.601639122567497, + "tokens_seen": 130678784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503510531594784, + "loss": 2.0268, + "theoretical_loss": 4.601023702731264, + "tokens_seen": 130809856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048501504513540625, + "loss": 1.8285, + "theoretical_loss": 4.600409071705312, + "tokens_seen": 130940928 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4414711594581604, + "objective/train/avg_token_score": 0.013736131601035595, + "objective/train/docs_used": 85560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.943097710609436, + "objective/train/original_loss": 4.5531487464904785, + "objective/train/theoretical_loss": 4.599795227690505, + "objective/train/tokens_used": 131072000, + "objective/train/unlikelihood_loss": 0.18566669523715973, + "theoretical_loss": 4.599795227690505, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849949849548646, + "loss": 1.7317, + "theoretical_loss": 4.599795227690505, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484974924774323, + "loss": 2.0678, + "theoretical_loss": 4.599182168893604, + "tokens_seen": 131203072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048495486459378134, + "loss": 2.0053, + "theoretical_loss": 4.5985698935272445, + "tokens_seen": 131334144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849348044132397, + "loss": 1.7232, + "theoretical_loss": 4.597958399809908, + "tokens_seen": 131465216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849147442326981, + "loss": 1.7742, + "theoretical_loss": 4.597347685965897, + "tokens_seen": 131596288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848946840521565, + "loss": 1.7775, + "theoretical_loss": 4.596737750225311, + "tokens_seen": 131727360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048487462387161484, + "loss": 1.6808, + "theoretical_loss": 4.596128590824026, + "tokens_seen": 131858432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848545636910732, + "loss": 1.6192, + "theoretical_loss": 4.595520206003663, + "tokens_seen": 131989504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848345035105316, + "loss": 1.7225, + "theoretical_loss": 4.594912594011566, + "tokens_seen": 132120576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048481444332999, + "loss": 1.9255, + "theoretical_loss": 4.594305753100782, + "tokens_seen": 132251648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048479438314944835, + "loss": 1.7942, + "theoretical_loss": 4.59369968153003, + "tokens_seen": 132382720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847743229689067, + "loss": 1.7619, + "theoretical_loss": 4.593094377563681, + "tokens_seen": 132513792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048475426278836507, + "loss": 1.7052, + "theoretical_loss": 4.592489839471735, + "tokens_seen": 132644864 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4741568863391876, + "objective/train/avg_token_score": 0.008817518129944801, + "objective/train/docs_used": 86181, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0522453784942627, + "objective/train/original_loss": 4.405289649963379, + "objective/train/theoretical_loss": 4.592187857089571, + "objective/train/tokens_used": 132710400, + "objective/train/unlikelihood_loss": 0.15506720542907715, + "theoretical_loss": 4.592187857089571, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847342026078235, + "loss": 1.6979, + "theoretical_loss": 4.591886065529795, + "tokens_seen": 132775936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048471414242728185, + "loss": 1.6689, + "theoretical_loss": 4.591283054019041, + "tokens_seen": 132907008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846940822467402, + "loss": 1.4962, + "theoretical_loss": 4.590680803226213, + "tokens_seen": 133038080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846740220661986, + "loss": 1.6976, + "theoretical_loss": 4.590079311443583, + "tokens_seen": 133169152 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484653961885657, + "loss": 1.7468, + "theoretical_loss": 4.589478576968932, + "tokens_seen": 133300224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048463390170511535, + "loss": 1.7804, + "theoretical_loss": 4.588878598105527, + "tokens_seen": 133431296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846138415245737, + "loss": 1.6541, + "theoretical_loss": 4.588279373162101, + "tokens_seen": 133562368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845937813440321, + "loss": 1.8706, + "theoretical_loss": 4.587680900452824, + "tokens_seen": 133693440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048457372116349044, + "loss": 1.8239, + "theoretical_loss": 4.587083178297288, + "tokens_seen": 133824512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048455366098294886, + "loss": 1.8302, + "theoretical_loss": 4.586486205020474, + "tokens_seen": 133955584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845336008024072, + "loss": 1.772, + "theoretical_loss": 4.585889978952741, + "tokens_seen": 134086656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845135406218656, + "loss": 1.6578, + "theoretical_loss": 4.585294498429796, + "tokens_seen": 134217728 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.49590665102005005, + "objective/train/avg_token_score": 0.008106932044029236, + "objective/train/docs_used": 87535, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1396756172180176, + "objective/train/original_loss": 4.32695198059082, + "objective/train/theoretical_loss": 4.584699761792674, + "objective/train/tokens_used": 134348800, + "objective/train/unlikelihood_loss": 0.16700398921966553, + "theoretical_loss": 4.584699761792674, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048449348044132394, + "loss": 1.8098, + "theoretical_loss": 4.584699761792674, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048447342026078236, + "loss": 1.9279, + "theoretical_loss": 4.5841057673877135, + "tokens_seen": 134479872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844533600802407, + "loss": 1.8587, + "theoretical_loss": 4.5835125135665375, + "tokens_seen": 134610944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844332998996991, + "loss": 1.6187, + "theoretical_loss": 4.58291999868603, + "tokens_seen": 134742016 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048441323971915745, + "loss": 1.58, + "theoretical_loss": 4.582328221108318, + "tokens_seen": 134873088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048439317953861586, + "loss": 1.9187, + "theoretical_loss": 4.581737179200739, + "tokens_seen": 135004160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843731193580742, + "loss": 1.8012, + "theoretical_loss": 4.581146871335832, + "tokens_seen": 135135232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843530591775326, + "loss": 1.745, + "theoretical_loss": 4.5805572958913086, + "tokens_seen": 135266304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048433299899699095, + "loss": 1.8915, + "theoretical_loss": 4.579968451250032, + "tokens_seen": 135397376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843129388164493, + "loss": 1.745, + "theoretical_loss": 4.579380335800001, + "tokens_seen": 135528448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048429287863590773, + "loss": 1.6448, + "theoretical_loss": 4.5787929479343195, + "tokens_seen": 135659520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842728184553661, + "loss": 1.9537, + "theoretical_loss": 4.578206286051184, + "tokens_seen": 135790592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048425275827482445, + "loss": 1.8237, + "theoretical_loss": 4.577620348553859, + "tokens_seen": 135921664 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.3742668628692627, + "objective/train/avg_token_score": 0.01183388289064169, + "objective/train/docs_used": 88189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5616800785064697, + "objective/train/original_loss": 4.133111476898193, + "objective/train/theoretical_loss": 4.577327650952276, + "objective/train/tokens_used": 135987200, + "objective/train/unlikelihood_loss": 0.14664104580879211, + "theoretical_loss": 4.577327650952276, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048423269809428287, + "loss": 1.6142, + "theoretical_loss": 4.5770351338506545, + "tokens_seen": 136052736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842126379137413, + "loss": 1.8337, + "theoretical_loss": 4.57645064035491, + "tokens_seen": 136183808 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048419257773319965, + "loss": 1.8521, + "theoretical_loss": 4.575866866484967, + "tokens_seen": 136314880 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484172517552658, + "loss": 1.6806, + "theoretical_loss": 4.575283810664155, + "tokens_seen": 136445952 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048415245737211637, + "loss": 1.6517, + "theoretical_loss": 4.574701471320768, + "tokens_seen": 136577024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048413239719157473, + "loss": 1.5807, + "theoretical_loss": 4.574119846888045, + "tokens_seen": 136708096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048411233701103315, + "loss": 1.6995, + "theoretical_loss": 4.573538935804146, + "tokens_seen": 136839168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840922768304915, + "loss": 1.7618, + "theoretical_loss": 4.5729587365121365, + "tokens_seen": 136970240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840722166499499, + "loss": 1.7269, + "theoretical_loss": 4.572379247459969, + "tokens_seen": 137101312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048405215646940824, + "loss": 1.629, + "theoretical_loss": 4.571800467100456, + "tokens_seen": 137232384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048403209628886665, + "loss": 1.7945, + "theoretical_loss": 4.571222393891253, + "tokens_seen": 137363456 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484012036108325, + "loss": 1.6366, + "theoretical_loss": 4.570645026294844, + "tokens_seen": 137494528 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4571114480495453, + "objective/train/avg_token_score": 0.013456963934004307, + "objective/train/docs_used": 88907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0327019691467285, + "objective/train/original_loss": 4.131555557250977, + "objective/train/theoretical_loss": 4.570068362778516, + "objective/train/tokens_used": 137625600, + "objective/train/unlikelihood_loss": 0.16605950891971588, + "theoretical_loss": 4.570068362778516, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839919759277834, + "loss": 1.9336, + "theoretical_loss": 4.570068362778516, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397191574724174, + "loss": 1.7372, + "theoretical_loss": 4.569492401814339, + "tokens_seen": 137756672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839518555667001, + "loss": 1.7945, + "theoretical_loss": 4.568917141879149, + "tokens_seen": 137887744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839317953861585, + "loss": 1.8875, + "theoretical_loss": 4.568342581454532, + "tokens_seen": 138018816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839117352056169, + "loss": 1.5855, + "theoretical_loss": 4.567768719026797, + "tokens_seen": 138149888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048389167502507524, + "loss": 1.796, + "theoretical_loss": 4.567195553086961, + "tokens_seen": 138280960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838716148445336, + "loss": 1.8751, + "theoretical_loss": 4.566623082130729, + "tokens_seen": 138412032 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483851554663992, + "loss": 1.7671, + "theoretical_loss": 4.56605130465848, + "tokens_seen": 138543104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838314944834504, + "loss": 1.7724, + "theoretical_loss": 4.565480219175237, + "tokens_seen": 138674176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381143430290875, + "loss": 1.8898, + "theoretical_loss": 4.56490982419066, + "tokens_seen": 138805248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837913741223671, + "loss": 1.766, + "theoretical_loss": 4.564340118219022, + "tokens_seen": 138936320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377131394182547, + "loss": 1.6873, + "theoretical_loss": 4.563771099779187, + "tokens_seen": 139067392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837512537612839, + "loss": 1.7903, + "theoretical_loss": 4.563202767394597, + "tokens_seen": 139198464 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4491080045700073, + "objective/train/avg_token_score": 0.0068944301456213, + "objective/train/docs_used": 90119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.943935751914978, + "objective/train/original_loss": 4.3753509521484375, + "objective/train/theoretical_loss": 4.562918858012843, + "objective/train/tokens_used": 139264000, + "objective/train/unlikelihood_loss": 0.1535024493932724, + "theoretical_loss": 4.562918858012843, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373119358074225, + "loss": 1.6177, + "theoretical_loss": 4.562635119593255, + "tokens_seen": 139329536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837111334002006, + "loss": 1.5881, + "theoretical_loss": 4.5620681549076965, + "tokens_seen": 139460608 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483691073219659, + "loss": 1.8352, + "theoretical_loss": 4.561501871874984, + "tokens_seen": 139591680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836710130391174, + "loss": 1.7988, + "theoretical_loss": 4.560936269036679, + "tokens_seen": 139722752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048365095285857575, + "loss": 1.694, + "theoretical_loss": 4.560371344938831, + "tokens_seen": 139853824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836308926780341, + "loss": 1.5882, + "theoretical_loss": 4.559807098131953, + "tokens_seen": 139984896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836108324974925, + "loss": 1.6214, + "theoretical_loss": 4.559243527171011, + "tokens_seen": 140115968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359077231695084, + "loss": 1.8377, + "theoretical_loss": 4.558680630615397, + "tokens_seen": 140247040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357071213640926, + "loss": 1.8401, + "theoretical_loss": 4.558118407028921, + "tokens_seen": 140378112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835506519558676, + "loss": 1.6105, + "theoretical_loss": 4.557556854979786, + "tokens_seen": 140509184 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483530591775326, + "loss": 1.7269, + "theoretical_loss": 4.556995973040574, + "tokens_seen": 140640256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048351053159478434, + "loss": 1.6745, + "theoretical_loss": 4.556435759788229, + "tokens_seen": 140771328 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.3440249264240265, + "objective/train/avg_token_score": 0.03104650229215622, + "objective/train/docs_used": 90792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5081690549850464, + "objective/train/original_loss": 4.372777462005615, + "objective/train/theoretical_loss": 4.555876213804037, + "objective/train/tokens_used": 140902400, + "objective/train/unlikelihood_loss": 0.17843031883239746, + "theoretical_loss": 4.555876213804037, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349047141424276, + "loss": 1.682, + "theoretical_loss": 4.555876213804037, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834704112337011, + "loss": 1.8494, + "theoretical_loss": 4.555317333673611, + "tokens_seen": 141033472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834503510531595, + "loss": 1.6859, + "theoretical_loss": 4.5547591179868725, + "tokens_seen": 141164544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343029087261785, + "loss": 1.7201, + "theoretical_loss": 4.554201565338033, + "tokens_seen": 141295616 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048341023069207626, + "loss": 1.6618, + "theoretical_loss": 4.553644674325584, + "tokens_seen": 141426688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833901705115346, + "loss": 1.6429, + "theoretical_loss": 4.553088443552269, + "tokens_seen": 141557760 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483370110330993, + "loss": 1.6037, + "theoretical_loss": 4.552532871625077, + "tokens_seen": 141688832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048335005015045135, + "loss": 1.564, + "theoretical_loss": 4.551977957155217, + "tokens_seen": 141819904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833299899699097, + "loss": 1.7074, + "theoretical_loss": 4.551423698758111, + "tokens_seen": 141950976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048330992978936813, + "loss": 1.6408, + "theoretical_loss": 4.550870095053366, + "tokens_seen": 142082048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832898696088265, + "loss": 1.6711, + "theoretical_loss": 4.550317144664766, + "tokens_seen": 142213120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048326980942828485, + "loss": 1.649, + "theoretical_loss": 4.5497648462202545, + "tokens_seen": 142344192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832497492477432, + "loss": 1.714, + "theoretical_loss": 4.549213198351914, + "tokens_seen": 142475264 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.31085044145584106, + "objective/train/avg_token_score": 0.008633834309875965, + "objective/train/docs_used": 92175, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2824527025222778, + "objective/train/original_loss": 4.095202922821045, + "objective/train/theoretical_loss": 4.548937617957463, + "objective/train/tokens_used": 142540800, + "objective/train/unlikelihood_loss": 0.16536656022071838, + "theoretical_loss": 4.548937617957463, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048322968906720163, + "loss": 1.6286, + "theoretical_loss": 4.548662199695954, + "tokens_seen": 142606336 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048320962888666, + "loss": 1.7102, + "theoretical_loss": 4.548111848892693, + "tokens_seen": 142737408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048318956870611836, + "loss": 1.6976, + "theoretical_loss": 4.547562144586539, + "tokens_seen": 142868480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831695085255767, + "loss": 1.8329, + "theoretical_loss": 4.5470130854259825, + "tokens_seen": 142999552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831494483450351, + "loss": 1.8789, + "theoretical_loss": 4.546464670063569, + "tokens_seen": 143130624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831293881644935, + "loss": 1.9697, + "theoretical_loss": 4.545916897155894, + "tokens_seen": 143261696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048310932798395186, + "loss": 1.7363, + "theoretical_loss": 4.545369765363578, + "tokens_seen": 143392768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830892678034102, + "loss": 1.889, + "theoretical_loss": 4.544823273351257, + "tokens_seen": 143523840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830692076228686, + "loss": 1.6188, + "theoretical_loss": 4.544277419787566, + "tokens_seen": 143654912 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483049147442327, + "loss": 1.8209, + "theoretical_loss": 4.543732203345119, + "tokens_seen": 143785984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048302908726178536, + "loss": 1.6722, + "theoretical_loss": 4.543187622700497, + "tokens_seen": 143917056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830090270812437, + "loss": 1.6843, + "theoretical_loss": 4.542643676534234, + "tokens_seen": 144048128 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.35123410820961, + "objective/train/avg_token_score": 0.024587556719779968, + "objective/train/docs_used": 92798, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.466112732887268, + "objective/train/original_loss": 4.362775802612305, + "objective/train/theoretical_loss": 4.542100363530799, + "objective/train/tokens_used": 144179200, + "objective/train/unlikelihood_loss": 0.18418121337890625, + "theoretical_loss": 4.542100363530799, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829889669007021, + "loss": 1.8888, + "theoretical_loss": 4.542100363530799, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048296890672016045, + "loss": 1.8105, + "theoretical_loss": 4.5415576823785795, + "tokens_seen": 144310272 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294884653961887, + "loss": 1.7692, + "theoretical_loss": 4.541015631769872, + "tokens_seen": 144441344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292878635907723, + "loss": 1.7058, + "theoretical_loss": 4.540474210400859, + "tokens_seen": 144572416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829087261785356, + "loss": 1.8536, + "theoretical_loss": 4.5399334169716, + "tokens_seen": 144703488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048288866599799395, + "loss": 1.7873, + "theoretical_loss": 4.539393250186015, + "tokens_seen": 144834560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048286860581745237, + "loss": 1.7762, + "theoretical_loss": 4.538853708751866, + "tokens_seen": 144965632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048284854563691073, + "loss": 1.7058, + "theoretical_loss": 4.538314791380748, + "tokens_seen": 145096704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828284854563691, + "loss": 1.9382, + "theoretical_loss": 4.537776496788071, + "tokens_seen": 145227776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048280842527582746, + "loss": 1.8217, + "theoretical_loss": 4.537238823693045, + "tokens_seen": 145358848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827883650952858, + "loss": 1.7635, + "theoretical_loss": 4.536701770818665, + "tokens_seen": 145489920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048276830491474423, + "loss": 1.7094, + "theoretical_loss": 4.536165336891699, + "tokens_seen": 145620992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827482447342026, + "loss": 1.8869, + "theoretical_loss": 4.53562952064267, + "tokens_seen": 145752064 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4389662742614746, + "objective/train/avg_token_score": 0.00564260920509696, + "objective/train/docs_used": 94036, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9034613370895386, + "objective/train/original_loss": 4.32117223739624, + "objective/train/theoretical_loss": 4.535361843751696, + "objective/train/tokens_used": 145817600, + "objective/train/unlikelihood_loss": 0.2049456685781479, + "theoretical_loss": 4.535361843751696, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482728184553661, + "loss": 1.8672, + "theoretical_loss": 4.535094320805847, + "tokens_seen": 145883136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827081243731194, + "loss": 1.8281, + "theoretical_loss": 4.5345597361192205, + "tokens_seen": 146014208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826880641925778, + "loss": 1.8617, + "theoretical_loss": 4.5340257653244995, + "tokens_seen": 146145280 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266800401203615, + "loss": 1.8899, + "theoretical_loss": 4.533492407167093, + "tokens_seen": 146276352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826479438314945, + "loss": 1.6389, + "theoretical_loss": 4.5329596603960916, + "tokens_seen": 146407424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826278836509529, + "loss": 1.831, + "theoretical_loss": 4.532427523764261, + "tokens_seen": 146538496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048260782347041124, + "loss": 1.67, + "theoretical_loss": 4.5318959960280205, + "tokens_seen": 146669568 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258776328986966, + "loss": 1.4795, + "theoretical_loss": 4.531365075947434, + "tokens_seen": 146800640 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482567703109328, + "loss": 1.9673, + "theoretical_loss": 4.5308347622861955, + "tokens_seen": 146931712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825476429287864, + "loss": 1.6775, + "theoretical_loss": 4.530305053811611, + "tokens_seen": 147062784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252758274824474, + "loss": 1.8903, + "theoretical_loss": 4.529775949294593, + "tokens_seen": 147193856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048250752256770316, + "loss": 1.5669, + "theoretical_loss": 4.529247447509637, + "tokens_seen": 147324928 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.5044599175453186, + "objective/train/avg_token_score": 0.009645281359553337, + "objective/train/docs_used": 94726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1166269779205322, + "objective/train/original_loss": 4.383207321166992, + "objective/train/theoretical_loss": 4.528719547234816, + "objective/train/tokens_used": 147456000, + "objective/train/unlikelihood_loss": 0.12163429707288742, + "theoretical_loss": 4.528719547234816, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824874623871615, + "loss": 1.874, + "theoretical_loss": 4.528719547234816, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824674022066199, + "loss": 1.5731, + "theoretical_loss": 4.528192247251763, + "tokens_seen": 147587072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244734202607825, + "loss": 1.7182, + "theoretical_loss": 4.527665546345656, + "tokens_seen": 147718144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048242728184553666, + "loss": 1.6724, + "theoretical_loss": 4.527139443305209, + "tokens_seen": 147849216 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482407221664995, + "loss": 1.8344, + "theoretical_loss": 4.526613936922654, + "tokens_seen": 147980288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823871614844534, + "loss": 1.729, + "theoretical_loss": 4.526089025993732, + "tokens_seen": 148111360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048236710130391175, + "loss": 1.914, + "theoretical_loss": 4.525564709317678, + "tokens_seen": 148242432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823470411233701, + "loss": 1.8067, + "theoretical_loss": 4.525040985697203, + "tokens_seen": 148373504 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048232698094282853, + "loss": 1.7183, + "theoretical_loss": 4.524517853938489, + "tokens_seen": 148504576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823069207622869, + "loss": 1.6597, + "theoretical_loss": 4.523995312851174, + "tokens_seen": 148635648 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048228686058174525, + "loss": 1.8709, + "theoretical_loss": 4.52347336124833, + "tokens_seen": 148766720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822668004012036, + "loss": 1.8413, + "theoretical_loss": 4.522951997946466, + "tokens_seen": 148897792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224674022066203, + "loss": 1.938, + "theoretical_loss": 4.522431221765498, + "tokens_seen": 149028864 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.5625610947608948, + "objective/train/avg_token_score": 0.0176337081938982, + "objective/train/docs_used": 95869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.122063159942627, + "objective/train/original_loss": 4.070887088775635, + "objective/train/theoretical_loss": 4.522171053477507, + "objective/train/tokens_used": 149094400, + "objective/train/unlikelihood_loss": 0.15213339030742645, + "theoretical_loss": 4.522171053477507, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822266800401204, + "loss": 1.9062, + "theoretical_loss": 4.5219110315287505, + "tokens_seen": 149159936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048220661985957876, + "loss": 1.7105, + "theoretical_loss": 4.521391426062934, + "tokens_seen": 149291008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821865596790371, + "loss": 1.8112, + "theoretical_loss": 4.520872404198139, + "tokens_seen": 149422080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821664994984955, + "loss": 1.6997, + "theoretical_loss": 4.520353964767814, + "tokens_seen": 149553152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821464393179539, + "loss": 1.6329, + "theoretical_loss": 4.519836106608768, + "tokens_seen": 149684224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048212637913741226, + "loss": 1.7948, + "theoretical_loss": 4.519318828561142, + "tokens_seen": 149815296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821063189568706, + "loss": 1.9672, + "theoretical_loss": 4.518802129468405, + "tokens_seen": 149946368 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482086258776329, + "loss": 1.7178, + "theoretical_loss": 4.518286008177341, + "tokens_seen": 150077440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820661985957874, + "loss": 1.8528, + "theoretical_loss": 4.517770463538038, + "tokens_seen": 150208512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204613841524576, + "loss": 1.7364, + "theoretical_loss": 4.517255494403868, + "tokens_seen": 150339584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820260782347041, + "loss": 1.7236, + "theoretical_loss": 4.516741099631485, + "tokens_seen": 150470656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820060180541625, + "loss": 1.6208, + "theoretical_loss": 4.5162272780808035, + "tokens_seen": 150601728 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4750122129917145, + "objective/train/avg_token_score": 0.008325014263391495, + "objective/train/docs_used": 96442, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0012364387512207, + "objective/train/original_loss": 4.350079536437988, + "objective/train/theoretical_loss": 4.515714028614996, + "objective/train/tokens_used": 150732800, + "objective/train/unlikelihood_loss": 0.13368818163871765, + "theoretical_loss": 4.515714028614996, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048198595787362085, + "loss": 1.7642, + "theoretical_loss": 4.515714028614996, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048196589769307927, + "loss": 1.986, + "theoretical_loss": 4.515201350100471, + "tokens_seen": 150863872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048194583751253763, + "loss": 1.6876, + "theoretical_loss": 4.5146892414068684, + "tokens_seen": 150994944 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481925777331996, + "loss": 1.3922, + "theoretical_loss": 4.514177701407042, + "tokens_seen": 151126016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048190571715145435, + "loss": 1.7374, + "theoretical_loss": 4.513666728977054, + "tokens_seen": 151257088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048188565697091277, + "loss": 1.7462, + "theoretical_loss": 4.513156322996155, + "tokens_seen": 151388160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048186559679037113, + "loss": 1.5728, + "theoretical_loss": 4.5126464823467805, + "tokens_seen": 151519232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818455366098295, + "loss": 1.7909, + "theoretical_loss": 4.5121372059145335, + "tokens_seen": 151650304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048182547642928786, + "loss": 1.5978, + "theoretical_loss": 4.511628492588174, + "tokens_seen": 151781376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818054162487462, + "loss": 1.6053, + "theoretical_loss": 4.511120341259608, + "tokens_seen": 151912448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048178535606820463, + "loss": 1.8598, + "theoretical_loss": 4.510612750823878, + "tokens_seen": 152043520 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481765295887663, + "loss": 1.864, + "theoretical_loss": 4.510105720179144, + "tokens_seen": 152174592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048174523570712136, + "loss": 1.7646, + "theoretical_loss": 4.509599248226683, + "tokens_seen": 152305664 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.5002443790435791, + "objective/train/avg_token_score": 0.033866036683321, + "objective/train/docs_used": 97778, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1443092823028564, + "objective/train/original_loss": 4.667900562286377, + "objective/train/theoretical_loss": 4.509346221417552, + "objective/train/tokens_used": 152371200, + "objective/train/unlikelihood_loss": 0.1134328842163086, + "theoretical_loss": 4.509346221417552, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817251755265797, + "loss": 1.7244, + "theoretical_loss": 4.509093333870869, + "tokens_seen": 152436736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048170511534603814, + "loss": 1.5927, + "theoretical_loss": 4.508587976019164, + "tokens_seen": 152567808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816850551654965, + "loss": 1.6639, + "theoretical_loss": 4.508083173582105, + "tokens_seen": 152698880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048166499498495486, + "loss": 1.8064, + "theoretical_loss": 4.5075789254733, + "tokens_seen": 152829952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816449348044132, + "loss": 1.6705, + "theoretical_loss": 4.507075230609407, + "tokens_seen": 152961024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816248746238716, + "loss": 1.6887, + "theoretical_loss": 4.506572087910127, + "tokens_seen": 153092096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048160481444333, + "loss": 1.7163, + "theoretical_loss": 4.506069496298198, + "tokens_seen": 153223168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048158475426278837, + "loss": 1.5463, + "theoretical_loss": 4.505567454699373, + "tokens_seen": 153354240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048156469408224673, + "loss": 1.7644, + "theoretical_loss": 4.505065962042418, + "tokens_seen": 153485312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815446339017051, + "loss": 1.5152, + "theoretical_loss": 4.504565017259097, + "tokens_seen": 153616384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815245737211635, + "loss": 1.6447, + "theoretical_loss": 4.504064619284163, + "tokens_seen": 153747456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048150451354062187, + "loss": 1.4444, + "theoretical_loss": 4.5035647670553445, + "tokens_seen": 153878528 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.584799587726593, + "objective/train/avg_token_score": 0.008578718639910221, + "objective/train/docs_used": 98099, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.4139657020568848, + "objective/train/original_loss": 4.249912261962891, + "objective/train/theoretical_loss": 4.503065459513339, + "objective/train/tokens_used": 154009600, + "objective/train/unlikelihood_loss": 0.09866306930780411, + "theoretical_loss": 4.503065459513339, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048148445336008023, + "loss": 1.8128, + "theoretical_loss": 4.503065459513339, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814643931795386, + "loss": 1.8889, + "theoretical_loss": 4.502566695601795, + "tokens_seen": 154140672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048144433299899696, + "loss": 1.8156, + "theoretical_loss": 4.502068474267309, + "tokens_seen": 154271744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048142427281845537, + "loss": 1.8757, + "theoretical_loss": 4.501570794459411, + "tokens_seen": 154402816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048140421263791373, + "loss": 1.8008, + "theoretical_loss": 4.501073655130554, + "tokens_seen": 154533888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813841524573721, + "loss": 1.781, + "theoretical_loss": 4.500577055236104, + "tokens_seen": 154664960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048136409227683046, + "loss": 1.7119, + "theoretical_loss": 4.500080993734329, + "tokens_seen": 154796032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813440320962889, + "loss": 1.8122, + "theoretical_loss": 4.499585469586387, + "tokens_seen": 154927104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048132397191574724, + "loss": 1.9573, + "theoretical_loss": 4.499090481756321, + "tokens_seen": 155058176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813039117352056, + "loss": 1.6949, + "theoretical_loss": 4.498596029211041, + "tokens_seen": 155189248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048128385155466396, + "loss": 1.8511, + "theoretical_loss": 4.4981021109203185, + "tokens_seen": 155320320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812637913741223, + "loss": 1.716, + "theoretical_loss": 4.497608725856776, + "tokens_seen": 155451392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048124373119358074, + "loss": 1.7856, + "theoretical_loss": 4.497115872995876, + "tokens_seen": 155582464 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4549120366573334, + "objective/train/avg_token_score": 0.0024577039293944836, + "objective/train/docs_used": 99496, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9255015850067139, + "objective/train/original_loss": 4.110994338989258, + "objective/train/theoretical_loss": 4.496869645822022, + "objective/train/tokens_used": 155648000, + "objective/train/unlikelihood_loss": 0.1798098087310791, + "theoretical_loss": 4.496869645822022, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048122367101303916, + "loss": 1.6885, + "theoretical_loss": 4.496623551315908, + "tokens_seen": 155713536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812036108324975, + "loss": 1.7518, + "theoretical_loss": 4.496131759797984, + "tokens_seen": 155844608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811835506519559, + "loss": 1.696, + "theoretical_loss": 4.495640497426023, + "tokens_seen": 155975680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811634904714143, + "loss": 1.7442, + "theoretical_loss": 4.4951497631867445, + "tokens_seen": 156106752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048114343029087266, + "loss": 1.6777, + "theoretical_loss": 4.4946595560696565, + "tokens_seen": 156237824 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481123370110331, + "loss": 1.6765, + "theoretical_loss": 4.494169875067046, + "tokens_seen": 156368896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811033099297894, + "loss": 1.6892, + "theoretical_loss": 4.493680719173968, + "tokens_seen": 156499968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810832497492478, + "loss": 1.5711, + "theoretical_loss": 4.493192087388239, + "tokens_seen": 156631040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048106318956870616, + "loss": 1.6435, + "theoretical_loss": 4.4927039787104235, + "tokens_seen": 156762112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810431293881645, + "loss": 1.4946, + "theoretical_loss": 4.492216392143826, + "tokens_seen": 156893184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810230692076229, + "loss": 1.9506, + "theoretical_loss": 4.49172932669448, + "tokens_seen": 157024256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048100300902708125, + "loss": 1.5955, + "theoretical_loss": 4.491242781371138, + "tokens_seen": 157155328 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.3397482931613922, + "objective/train/avg_token_score": 0.02855628915131092, + "objective/train/docs_used": 100000, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4549492597579956, + "objective/train/original_loss": 4.49015998840332, + "objective/train/theoretical_loss": 4.4907567551852665, + "objective/train/tokens_used": 157286400, + "objective/train/unlikelihood_loss": 0.11378736048936844, + "theoretical_loss": 4.4907567551852665, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048098294884653967, + "loss": 1.5109, + "theoretical_loss": 4.4907567551852665, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048096288866599803, + "loss": 1.7852, + "theoretical_loss": 4.490271247151027, + "tokens_seen": 157417472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809428284854564, + "loss": 1.6, + "theoretical_loss": 4.489786256285276, + "tokens_seen": 157548544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048092276830491475, + "loss": 1.8271, + "theoretical_loss": 4.489301781607551, + "tokens_seen": 157679616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048090270812437317, + "loss": 1.8531, + "theoretical_loss": 4.48881782214006, + "tokens_seen": 157810688 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048088264794383153, + "loss": 1.7978, + "theoretical_loss": 4.488334376907673, + "tokens_seen": 157941760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808625877632899, + "loss": 1.5617, + "theoretical_loss": 4.487851444937916, + "tokens_seen": 158072832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084252758274826, + "loss": 1.7742, + "theoretical_loss": 4.487369025260954, + "tokens_seen": 158203904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808224674022066, + "loss": 1.7228, + "theoretical_loss": 4.48688711690959, + "tokens_seen": 158334976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048080240722166503, + "loss": 1.819, + "theoretical_loss": 4.48640571891925, + "tokens_seen": 158466048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807823470411234, + "loss": 1.7198, + "theoretical_loss": 4.485924830327974, + "tokens_seen": 158597120 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048076228686058176, + "loss": 1.7289, + "theoretical_loss": 4.485444450176413, + "tokens_seen": 158728192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807422266800401, + "loss": 1.694, + "theoretical_loss": 4.484964577507808, + "tokens_seen": 158859264 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.47953322529792786, + "objective/train/avg_token_score": 0.00853870902210474, + "objective/train/docs_used": 101476, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9475176334381104, + "objective/train/original_loss": 4.236670970916748, + "objective/train/theoretical_loss": 4.484724831181337, + "objective/train/tokens_used": 158924800, + "objective/train/unlikelihood_loss": 0.1266087293624878, + "theoretical_loss": 4.484724831181337, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072216649949854, + "loss": 1.8989, + "theoretical_loss": 4.4844852113679945, + "tokens_seen": 158990336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807021063189569, + "loss": 1.5987, + "theoretical_loss": 4.484006350805385, + "tokens_seen": 159121408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068204613841526, + "loss": 1.7375, + "theoretical_loss": 4.483527994870958, + "tokens_seen": 159252480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806619859578736, + "loss": 1.5585, + "theoretical_loss": 4.483050142618255, + "tokens_seen": 159383552 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480641925777332, + "loss": 1.7186, + "theoretical_loss": 4.482572793103373, + "tokens_seen": 159514624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806218655967904, + "loss": 1.6185, + "theoretical_loss": 4.482095945384946, + "tokens_seen": 159645696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048060180541624877, + "loss": 1.6661, + "theoretical_loss": 4.4816195985241425, + "tokens_seen": 159776768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058174523570713, + "loss": 1.7461, + "theoretical_loss": 4.481143751584659, + "tokens_seen": 159907840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805616850551655, + "loss": 1.8502, + "theoretical_loss": 4.480668403632706, + "tokens_seen": 160038912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805416248746239, + "loss": 1.7936, + "theoretical_loss": 4.480193553736999, + "tokens_seen": 160169984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052156469408227, + "loss": 1.9024, + "theoretical_loss": 4.479719200968757, + "tokens_seen": 160301056 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048050150451354063, + "loss": 1.7003, + "theoretical_loss": 4.479245344401685, + "tokens_seen": 160432128 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.53665691614151, + "objective/train/avg_token_score": 0.011694345623254776, + "objective/train/docs_used": 102054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.992788553237915, + "objective/train/original_loss": 3.9252665042877197, + "objective/train/theoretical_loss": 4.478771983111967, + "objective/train/tokens_used": 160563200, + "objective/train/unlikelihood_loss": 0.1462445855140686, + "theoretical_loss": 4.478771983111967, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480481444332999, + "loss": 1.6986, + "theoretical_loss": 4.478771983111967, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048046138415245736, + "loss": 1.4835, + "theoretical_loss": 4.478299116178265, + "tokens_seen": 160694272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048044132397191577, + "loss": 1.5806, + "theoretical_loss": 4.4778267426817, + "tokens_seen": 160825344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042126379137413, + "loss": 1.7454, + "theoretical_loss": 4.47735486170585, + "tokens_seen": 160956416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804012036108325, + "loss": 1.6559, + "theoretical_loss": 4.4768834723367394, + "tokens_seen": 161087488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038114343029086, + "loss": 1.7753, + "theoretical_loss": 4.476412573662829, + "tokens_seen": 161218560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803610832497493, + "loss": 1.6249, + "theoretical_loss": 4.475942164775013, + "tokens_seen": 161349632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034102306920764, + "loss": 1.6971, + "theoretical_loss": 4.475472244766601, + "tokens_seen": 161480704 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480320962888666, + "loss": 1.6015, + "theoretical_loss": 4.47500281273332, + "tokens_seen": 161611776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048030090270812436, + "loss": 1.8606, + "theoretical_loss": 4.474533867773299, + "tokens_seen": 161742848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802808425275827, + "loss": 1.615, + "theoretical_loss": 4.474065408987063, + "tokens_seen": 161873920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048026078234704114, + "loss": 1.5313, + "theoretical_loss": 4.473597435477526, + "tokens_seen": 162004992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802407221664995, + "loss": 1.8292, + "theoretical_loss": 4.473129946349982, + "tokens_seen": 162136064 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.436950147151947, + "objective/train/avg_token_score": 0.005409137345850468, + "objective/train/docs_used": 102718, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9111195802688599, + "objective/train/original_loss": 4.506403923034668, + "objective/train/theoretical_loss": 4.472896383150508, + "objective/train/tokens_used": 162201600, + "objective/train/unlikelihood_loss": 0.16208896040916443, + "theoretical_loss": 4.472896383150508, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048022066198595786, + "loss": 1.6402, + "theoretical_loss": 4.472662940712091, + "tokens_seen": 162267136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020060180541623, + "loss": 1.5722, + "theoretical_loss": 4.472196417673883, + "tokens_seen": 162398208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048018054162487464, + "loss": 1.6823, + "theoretical_loss": 4.471730376347738, + "tokens_seen": 162529280 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480160481444333, + "loss": 1.6869, + "theoretical_loss": 4.471264815848384, + "tokens_seen": 162660352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014042126379137, + "loss": 1.794, + "theoretical_loss": 4.470799735292889, + "tokens_seen": 162791424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048012036108324973, + "loss": 1.7949, + "theoretical_loss": 4.470335133800649, + "tokens_seen": 162922496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048010030090270815, + "loss": 1.6686, + "theoretical_loss": 4.469871010493383, + "tokens_seen": 163053568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800802407221665, + "loss": 1.7866, + "theoretical_loss": 4.4694073644951295, + "tokens_seen": 163184640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048006018054162487, + "loss": 1.6188, + "theoretical_loss": 4.468944194932225, + "tokens_seen": 163315712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004012036108323, + "loss": 1.7558, + "theoretical_loss": 4.46848150093331, + "tokens_seen": 163446784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800200601805416, + "loss": 1.9446, + "theoretical_loss": 4.468019281629316, + "tokens_seen": 163577856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 1.9535, + "theoretical_loss": 4.467557536153457, + "tokens_seen": 163708928 + }, + { + "debugging/length": 491.6, + "debugging/num_segments": 5, + "epoch": 0.05, + "objective/train/aligned_token_count": 0.416177898645401, + "objective/train/avg_token_score": 0.005864487960934639, + "objective/train/docs_used": 104069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5953806638717651, + "objective/train/original_loss": 3.946671962738037, + "objective/train/theoretical_loss": 4.467096263641219, + "objective/train/tokens_used": 163840000, + "objective/train/unlikelihood_loss": 0.20240885019302368, + "theoretical_loss": 4.467096263641219, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799799398194584, + "loss": 1.8006, + "theoretical_loss": 4.467096263641219, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047995987963891674, + "loss": 1.7427, + "theoretical_loss": 4.466635463230359, + "tokens_seen": 163971072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799398194583751, + "loss": 1.7973, + "theoretical_loss": 4.466175134060894, + "tokens_seen": 164102144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799197592778335, + "loss": 1.7691, + "theoretical_loss": 4.4657152752750875, + "tokens_seen": 164233216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798996990972919, + "loss": 1.8554, + "theoretical_loss": 4.465255886017452, + "tokens_seen": 164364288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047987963891675024, + "loss": 1.9336, + "theoretical_loss": 4.464796965434738, + "tokens_seen": 164495360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798595787362086, + "loss": 1.5818, + "theoretical_loss": 4.464338512675919, + "tokens_seen": 164626432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983951855566696, + "loss": 1.8526, + "theoretical_loss": 4.4638805268921935, + "tokens_seen": 164757504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798194583751254, + "loss": 1.8442, + "theoretical_loss": 4.463423007236974, + "tokens_seen": 164888576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047979939819458374, + "loss": 1.418, + "theoretical_loss": 4.462965952865879, + "tokens_seen": 165019648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797793380140421, + "loss": 1.7218, + "theoretical_loss": 4.462509362936723, + "tokens_seen": 165150720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975927783350047, + "loss": 1.7421, + "theoretical_loss": 4.462053236609516, + "tokens_seen": 165281792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797392176529589, + "loss": 1.7061, + "theoretical_loss": 4.461597573046449, + "tokens_seen": 165412864 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4883309006690979, + "objective/train/avg_token_score": 0.004766791593283415, + "objective/train/docs_used": 105300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9741791486740112, + "objective/train/original_loss": 3.7647905349731445, + "objective/train/theoretical_loss": 4.461369914540247, + "objective/train/tokens_used": 165478400, + "objective/train/unlikelihood_loss": 0.15376895666122437, + "theoretical_loss": 4.461369914540247, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797191574724173, + "loss": 1.7765, + "theoretical_loss": 4.4611423714118885, + "tokens_seen": 165543936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047969909729187566, + "loss": 2.1214, + "theoretical_loss": 4.460687630872371, + "tokens_seen": 165675008 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479679037111334, + "loss": 1.8148, + "theoretical_loss": 4.460233350596599, + "tokens_seen": 165806080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796589769307924, + "loss": 1.8356, + "theoretical_loss": 4.459779529755423, + "tokens_seen": 165937152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796389167502508, + "loss": 1.847, + "theoretical_loss": 4.459326167521844, + "tokens_seen": 166068224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047961885656970917, + "loss": 1.5851, + "theoretical_loss": 4.458873263071002, + "tokens_seen": 166199296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047959879638916753, + "loss": 1.87, + "theoretical_loss": 4.458420815580169, + "tokens_seen": 166330368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795787362086259, + "loss": 1.7718, + "theoretical_loss": 4.457968824228743, + "tokens_seen": 166461440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795586760280843, + "loss": 1.9016, + "theoretical_loss": 4.4575172881982414, + "tokens_seen": 166592512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953861584754267, + "loss": 1.5086, + "theoretical_loss": 4.457066206672291, + "tokens_seen": 166723584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047951855566700103, + "loss": 1.8205, + "theoretical_loss": 4.456615578836625, + "tokens_seen": 166854656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794984954864594, + "loss": 1.617, + "theoretical_loss": 4.45616540387907, + "tokens_seen": 166985728 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.5096529722213745, + "objective/train/avg_token_score": 0.009488563984632492, + "objective/train/docs_used": 106148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0118892192840576, + "objective/train/original_loss": 4.2763590812683105, + "objective/train/theoretical_loss": 4.455715680989545, + "objective/train/tokens_used": 167116800, + "objective/train/unlikelihood_loss": 0.12322863936424255, + "theoretical_loss": 4.455715680989545, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047947843530591776, + "loss": 1.6269, + "theoretical_loss": 4.455715680989545, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047945837512537617, + "loss": 1.809, + "theoretical_loss": 4.4552664093600525, + "tokens_seen": 167247872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943831494483453, + "loss": 1.7982, + "theoretical_loss": 4.454817588184669, + "tokens_seen": 167378944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794182547642929, + "loss": 1.6824, + "theoretical_loss": 4.454369216659542, + "tokens_seen": 167510016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939819458375126, + "loss": 1.82, + "theoretical_loss": 4.453921293982877, + "tokens_seen": 167641088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793781344032097, + "loss": 1.4183, + "theoretical_loss": 4.453473819354942, + "tokens_seen": 167772160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047935807422266804, + "loss": 1.6876, + "theoretical_loss": 4.453026791978045, + "tokens_seen": 167903232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793380140421264, + "loss": 1.4265, + "theoretical_loss": 4.452580211056542, + "tokens_seen": 168034304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047931795386158476, + "loss": 1.7324, + "theoretical_loss": 4.45213407579682, + "tokens_seen": 168165376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792978936810431, + "loss": 1.7566, + "theoretical_loss": 4.451688385407296, + "tokens_seen": 168296448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047927783350050154, + "loss": 1.6371, + "theoretical_loss": 4.4512431390984055, + "tokens_seen": 168427520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792577733199599, + "loss": 1.6714, + "theoretical_loss": 4.450798336082601, + "tokens_seen": 168558592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923771313941827, + "loss": 1.6945, + "theoretical_loss": 4.450353975574341, + "tokens_seen": 168689664 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.3742057681083679, + "objective/train/avg_token_score": 0.02583669312298298, + "objective/train/docs_used": 106755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.461486577987671, + "objective/train/original_loss": 4.418996334075928, + "objective/train/theoretical_loss": 4.450131961015606, + "objective/train/tokens_used": 168755200, + "objective/train/unlikelihood_loss": 0.12685106694698334, + "theoretical_loss": 4.450131961015606, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047921765295887663, + "loss": 1.7889, + "theoretical_loss": 4.449910056790086, + "tokens_seen": 168820736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919759277833504, + "loss": 1.6048, + "theoretical_loss": 4.4494665789482895, + "tokens_seen": 168951808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791775325977934, + "loss": 1.5793, + "theoretical_loss": 4.449023541269395, + "tokens_seen": 169082880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047915747241725177, + "loss": 1.6751, + "theoretical_loss": 4.448580942975825, + "tokens_seen": 169213952 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913741223671013, + "loss": 1.7455, + "theoretical_loss": 4.448138783291979, + "tokens_seen": 169345024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047911735205616855, + "loss": 1.7699, + "theoretical_loss": 4.44769706144422, + "tokens_seen": 169476096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790972918756269, + "loss": 1.6299, + "theoretical_loss": 4.447255776660878, + "tokens_seen": 169607168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047907723169508527, + "loss": 1.5542, + "theoretical_loss": 4.446814928172234, + "tokens_seen": 169738240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905717151454363, + "loss": 1.7944, + "theoretical_loss": 4.446374515210521, + "tokens_seen": 169869312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479037111334002, + "loss": 1.6775, + "theoretical_loss": 4.445934537009911, + "tokens_seen": 170000384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790170511534604, + "loss": 1.8059, + "theoretical_loss": 4.445494992806513, + "tokens_seen": 170131456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789969909729188, + "loss": 1.8668, + "theoretical_loss": 4.445055881838365, + "tokens_seen": 170262528 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.41752198338508606, + "objective/train/avg_token_score": 0.02483731135725975, + "objective/train/docs_used": 107830, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6519962549209595, + "objective/train/original_loss": 4.723080635070801, + "objective/train/theoretical_loss": 4.44461720334543, + "objective/train/tokens_used": 170393600, + "objective/train/unlikelihood_loss": 0.11609357595443726, + "theoretical_loss": 4.44461720334543, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047897693079237714, + "loss": 1.7401, + "theoretical_loss": 4.44461720334543, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789568706118355, + "loss": 1.6675, + "theoretical_loss": 4.444178956569585, + "tokens_seen": 170524672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789368104312939, + "loss": 1.7224, + "theoretical_loss": 4.44374114075462, + "tokens_seen": 170655744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789167502507523, + "loss": 1.6234, + "theoretical_loss": 4.443303755146225, + "tokens_seen": 170786816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047889669007021064, + "loss": 1.6098, + "theoretical_loss": 4.442866798991993, + "tokens_seen": 170917888 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478876629889669, + "loss": 1.749, + "theoretical_loss": 4.442430271541404, + "tokens_seen": 171048960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047885656970912736, + "loss": 1.6635, + "theoretical_loss": 4.4419941720458285, + "tokens_seen": 171180032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788365095285858, + "loss": 1.8943, + "theoretical_loss": 4.441558499758511, + "tokens_seen": 171311104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047881644934804414, + "loss": 1.6301, + "theoretical_loss": 4.441123253934572, + "tokens_seen": 171442176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787963891675025, + "loss": 1.5459, + "theoretical_loss": 4.440688433830999, + "tokens_seen": 171573248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047877632898696087, + "loss": 1.7699, + "theoretical_loss": 4.440254038706639, + "tokens_seen": 171704320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787562688064193, + "loss": 1.6063, + "theoretical_loss": 4.439820067822195, + "tokens_seen": 171835392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047873620862587765, + "loss": 1.7779, + "theoretical_loss": 4.439386520440218, + "tokens_seen": 171966464 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.3144550323486328, + "objective/train/avg_token_score": 0.00989236868917942, + "objective/train/docs_used": 108471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1930038928985596, + "objective/train/original_loss": 4.118610382080078, + "objective/train/theoretical_loss": 4.439169905332706, + "objective/train/tokens_used": 172032000, + "objective/train/unlikelihood_loss": 0.1678958386182785, + "theoretical_loss": 4.439169905332706, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478716148445336, + "loss": 1.4924, + "theoretical_loss": 4.438953395825102, + "tokens_seen": 172097536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047869608826479437, + "loss": 1.6309, + "theoretical_loss": 4.438520693243079, + "tokens_seen": 172228608 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047867602808425273, + "loss": 1.5903, + "theoretical_loss": 4.438088411962211, + "tokens_seen": 172359680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047865596790371115, + "loss": 1.5514, + "theoretical_loss": 4.437656551252381, + "tokens_seen": 172490752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786359077231695, + "loss": 1.8921, + "theoretical_loss": 4.437225110385297, + "tokens_seen": 172621824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786158475426279, + "loss": 1.6624, + "theoretical_loss": 4.436794088634477, + "tokens_seen": 172752896 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047859578736208624, + "loss": 1.653, + "theoretical_loss": 4.436363485275246, + "tokens_seen": 172883968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047857572718154465, + "loss": 1.6515, + "theoretical_loss": 4.435933299584729, + "tokens_seen": 173015040 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478555667001003, + "loss": 1.6443, + "theoretical_loss": 4.435503530841849, + "tokens_seen": 173146112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785356068204614, + "loss": 1.6424, + "theoretical_loss": 4.4350741783273175, + "tokens_seen": 173277184 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047851554663991974, + "loss": 1.7407, + "theoretical_loss": 4.434645241323629, + "tokens_seen": 173408256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784954864593781, + "loss": 1.6988, + "theoretical_loss": 4.434216719115057, + "tokens_seen": 173539328 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.38972383737564087, + "objective/train/avg_token_score": 0.009662693366408348, + "objective/train/docs_used": 109645, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5914331674575806, + "objective/train/original_loss": 4.367075443267822, + "objective/train/theoretical_loss": 4.433788610987646, + "objective/train/tokens_used": 173670400, + "objective/train/unlikelihood_loss": 0.1352538764476776, + "theoretical_loss": 4.433788610987646, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784754262788365, + "loss": 1.7224, + "theoretical_loss": 4.433788610987646, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784553660982949, + "loss": 1.7154, + "theoretical_loss": 4.433360916229209, + "tokens_seen": 173801472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047843530591775324, + "loss": 1.6758, + "theoretical_loss": 4.432933634129318, + "tokens_seen": 173932544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784152457372116, + "loss": 1.6071, + "theoretical_loss": 4.432506763979299, + "tokens_seen": 174063616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047839518555667, + "loss": 1.6384, + "theoretical_loss": 4.432080305072233, + "tokens_seen": 174194688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783751253761284, + "loss": 1.5316, + "theoretical_loss": 4.431654256702938, + "tokens_seen": 174325760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047835506519558675, + "loss": 1.614, + "theoretical_loss": 4.4312286181679745, + "tokens_seen": 174456832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783350050150451, + "loss": 1.9548, + "theoretical_loss": 4.430803388765636, + "tokens_seen": 174587904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047831494483450347, + "loss": 1.5284, + "theoretical_loss": 4.430378567795938, + "tokens_seen": 174718976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782948846539619, + "loss": 1.6997, + "theoretical_loss": 4.429954154560624, + "tokens_seen": 174850048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047827482447342025, + "loss": 1.6483, + "theoretical_loss": 4.429530148363151, + "tokens_seen": 174981120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782547642928786, + "loss": 1.5161, + "theoretical_loss": 4.429106548508685, + "tokens_seen": 175112192 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478234704112337, + "loss": 1.7931, + "theoretical_loss": 4.428683354304098, + "tokens_seen": 175243264 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.3965053856372833, + "objective/train/avg_token_score": 0.01913469098508358, + "objective/train/docs_used": 110166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7019623517990112, + "objective/train/original_loss": 4.4854278564453125, + "objective/train/theoretical_loss": 4.428471909104372, + "objective/train/tokens_used": 175308800, + "objective/train/unlikelihood_loss": 0.11818500608205795, + "theoretical_loss": 4.428471909104372, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047821464393179544, + "loss": 1.5788, + "theoretical_loss": 4.428260565057964, + "tokens_seen": 175374336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781945837512538, + "loss": 1.786, + "theoretical_loss": 4.427838180080547, + "tokens_seen": 175505408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047817452357071217, + "loss": 1.7903, + "theoretical_loss": 4.427416198683803, + "tokens_seen": 175636480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047815446339017053, + "loss": 1.7291, + "theoretical_loss": 4.42699462018137, + "tokens_seen": 175767552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047813440320962895, + "loss": 1.8156, + "theoretical_loss": 4.426573443888563, + "tokens_seen": 175898624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781143430290873, + "loss": 1.6172, + "theoretical_loss": 4.426152669122374, + "tokens_seen": 176029696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047809428284854567, + "loss": 1.5866, + "theoretical_loss": 4.425732295201455, + "tokens_seen": 176160768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047807422266800403, + "loss": 1.6032, + "theoretical_loss": 4.425312321446127, + "tokens_seen": 176291840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780541624874624, + "loss": 1.7259, + "theoretical_loss": 4.424892747178365, + "tokens_seen": 176422912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780341023069208, + "loss": 1.5495, + "theoretical_loss": 4.424473571721794, + "tokens_seen": 176553984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780140421263792, + "loss": 1.5823, + "theoretical_loss": 4.424054794401689, + "tokens_seen": 176685056 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799398194583754, + "loss": 1.5334, + "theoretical_loss": 4.42363641454496, + "tokens_seen": 176816128 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.27394917607307434, + "objective/train/avg_token_score": 0.024683641269803047, + "objective/train/docs_used": 110915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1538923978805542, + "objective/train/original_loss": 4.269205570220947, + "objective/train/theoretical_loss": 4.42321843148016, + "objective/train/tokens_used": 176947200, + "objective/train/unlikelihood_loss": 0.17650169134140015, + "theoretical_loss": 4.42321843148016, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779739217652959, + "loss": 1.5801, + "theoretical_loss": 4.42321843148016, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779538615847543, + "loss": 1.743, + "theoretical_loss": 4.422800844537466, + "tokens_seen": 177078272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779338014042127, + "loss": 1.7594, + "theoretical_loss": 4.422383653048685, + "tokens_seen": 177209344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047791374122367104, + "loss": 1.7924, + "theoretical_loss": 4.421966856347243, + "tokens_seen": 177340416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778936810431294, + "loss": 1.9118, + "theoretical_loss": 4.421550453768181, + "tokens_seen": 177471488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047787362086258776, + "loss": 1.6296, + "theoretical_loss": 4.42113444464815, + "tokens_seen": 177602560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778535606820462, + "loss": 1.512, + "theoretical_loss": 4.420718828325403, + "tokens_seen": 177733632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783350050150454, + "loss": 1.7201, + "theoretical_loss": 4.4203036041398, + "tokens_seen": 177864704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778134403209629, + "loss": 1.6327, + "theoretical_loss": 4.419888771432789, + "tokens_seen": 177995776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047779338014042127, + "loss": 1.5254, + "theoretical_loss": 4.419474329547413, + "tokens_seen": 178126848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777733199598797, + "loss": 1.8315, + "theoretical_loss": 4.419060277828295, + "tokens_seen": 178257920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775325977933805, + "loss": 1.6243, + "theoretical_loss": 4.4186466156216415, + "tokens_seen": 178388992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777331995987964, + "loss": 1.6067, + "theoretical_loss": 4.418233342275233, + "tokens_seen": 178520064 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4303519129753113, + "objective/train/avg_token_score": 0.005776857491582632, + "objective/train/docs_used": 112413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6707626581192017, + "objective/train/original_loss": 3.843381643295288, + "objective/train/theoretical_loss": 4.418026851221231, + "objective/train/tokens_used": 178585600, + "objective/train/unlikelihood_loss": 0.19406434893608093, + "theoretical_loss": 4.418026851221231, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771313941825477, + "loss": 1.5482, + "theoretical_loss": 4.4178204571384185, + "tokens_seen": 178651136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047769307923771313, + "loss": 1.5164, + "theoretical_loss": 4.417407959562116, + "tokens_seen": 178782208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047767301905717155, + "loss": 1.6927, + "theoretical_loss": 4.416995848898797, + "tokens_seen": 178913280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776529588766299, + "loss": 1.496, + "theoretical_loss": 4.416584124502495, + "tokens_seen": 179044352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776328986960883, + "loss": 1.5685, + "theoretical_loss": 4.41617278572879, + "tokens_seen": 179175424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761283851554664, + "loss": 1.4586, + "theoretical_loss": 4.415761831934808, + "tokens_seen": 179306496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047759277833500505, + "loss": 1.4191, + "theoretical_loss": 4.415351262479216, + "tokens_seen": 179437568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775727181544634, + "loss": 1.5665, + "theoretical_loss": 4.414941076722219, + "tokens_seen": 179568640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775526579739218, + "loss": 1.545, + "theoretical_loss": 4.414531274025548, + "tokens_seen": 179699712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753259779338014, + "loss": 1.5296, + "theoretical_loss": 4.414121853752466, + "tokens_seen": 179830784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775125376128385, + "loss": 1.5963, + "theoretical_loss": 4.413712815267752, + "tokens_seen": 179961856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774924774322969, + "loss": 1.8008, + "theoretical_loss": 4.4133041579377075, + "tokens_seen": 180092928 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.39986559748649597, + "objective/train/avg_token_score": 0.06428433954715729, + "objective/train/docs_used": 112982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4951305389404297, + "objective/train/original_loss": 4.130880355834961, + "objective/train/theoretical_loss": 4.412895881130142, + "objective/train/tokens_used": 180224000, + "objective/train/unlikelihood_loss": 0.13160917162895203, + "theoretical_loss": 4.412895881130142, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774724172517553, + "loss": 1.6337, + "theoretical_loss": 4.412895881130142, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745235707121364, + "loss": 1.658, + "theoretical_loss": 4.412487984214373, + "tokens_seen": 180355072 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477432296890672, + "loss": 1.5547, + "theoretical_loss": 4.412080466561221, + "tokens_seen": 180486144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774122367101304, + "loss": 1.7956, + "theoretical_loss": 4.411673327543005, + "tokens_seen": 180617216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773921765295888, + "loss": 1.7189, + "theoretical_loss": 4.411266566533539, + "tokens_seen": 180748288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047737211634904715, + "loss": 1.6131, + "theoretical_loss": 4.41086018290812, + "tokens_seen": 180879360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773520561685055, + "loss": 1.7809, + "theoretical_loss": 4.410454176043537, + "tokens_seen": 181010432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733199598796387, + "loss": 1.5577, + "theoretical_loss": 4.410048545318052, + "tokens_seen": 181141504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773119358074223, + "loss": 1.4906, + "theoretical_loss": 4.409643290111404, + "tokens_seen": 181272576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729187562688065, + "loss": 1.5827, + "theoretical_loss": 4.409238409804804, + "tokens_seen": 181403648 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477271815446339, + "loss": 1.6267, + "theoretical_loss": 4.408833903780926, + "tokens_seen": 181534720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772517552657974, + "loss": 1.6512, + "theoretical_loss": 4.408429771423909, + "tokens_seen": 181665792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772316950852558, + "loss": 1.7169, + "theoretical_loss": 4.408026012119344, + "tokens_seen": 181796864 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3759775161743164, + "objective/train/avg_token_score": 0.011839372105896473, + "objective/train/docs_used": 114216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4656167030334473, + "objective/train/original_loss": 4.162503719329834, + "objective/train/theoretical_loss": 4.407824272170128, + "objective/train/tokens_used": 181862400, + "objective/train/unlikelihood_loss": 0.16261805593967438, + "theoretical_loss": 4.407824272170128, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047721163490471415, + "loss": 1.7268, + "theoretical_loss": 4.407622625254279, + "tokens_seen": 181927936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771915747241725, + "loss": 1.6943, + "theoretical_loss": 4.407219610217206, + "tokens_seen": 182059008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771715145436309, + "loss": 1.7274, + "theoretical_loss": 4.406816966398064, + "tokens_seen": 182190080 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047715145436308924, + "loss": 1.539, + "theoretical_loss": 4.4064146931882275, + "tokens_seen": 182321152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713139418254766, + "loss": 1.6115, + "theoretical_loss": 4.406012789980506, + "tokens_seen": 182452224 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477111334002006, + "loss": 1.6946, + "theoretical_loss": 4.405611256169143, + "tokens_seen": 182583296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770912738214644, + "loss": 1.69, + "theoretical_loss": 4.405210091149802, + "tokens_seen": 182714368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047707121364092274, + "loss": 1.7164, + "theoretical_loss": 4.404809294319572, + "tokens_seen": 182845440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047705115346038116, + "loss": 1.7673, + "theoretical_loss": 4.404408865076955, + "tokens_seen": 182976512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770310932798395, + "loss": 1.7797, + "theoretical_loss": 4.404008802821871, + "tokens_seen": 183107584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770110330992979, + "loss": 1.7046, + "theoretical_loss": 4.403609106955645, + "tokens_seen": 183238656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047699097291875625, + "loss": 1.5983, + "theoretical_loss": 4.403209776881004, + "tokens_seen": 183369728 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4590664803981781, + "objective/train/avg_token_score": 0.02654801867902279, + "objective/train/docs_used": 114775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.934823751449585, + "objective/train/original_loss": 4.4886064529418945, + "objective/train/theoretical_loss": 4.4028108120020795, + "objective/train/tokens_used": 183500800, + "objective/train/unlikelihood_loss": 0.1138959601521492, + "theoretical_loss": 4.4028108120020795, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047697091273821466, + "loss": 1.7135, + "theoretical_loss": 4.4028108120020795, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476950852557673, + "loss": 1.6734, + "theoretical_loss": 4.4024122117243945, + "tokens_seen": 183631872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769307923771314, + "loss": 1.8335, + "theoretical_loss": 4.4020139754548655, + "tokens_seen": 183762944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047691073219658975, + "loss": 1.6858, + "theoretical_loss": 4.4016161026017935, + "tokens_seen": 183894016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768906720160481, + "loss": 1.7323, + "theoretical_loss": 4.401218592574865, + "tokens_seen": 184025088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047687061183550653, + "loss": 1.8249, + "theoretical_loss": 4.400821444785143, + "tokens_seen": 184156160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768505516549649, + "loss": 1.588, + "theoretical_loss": 4.400424658645065, + "tokens_seen": 184287232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047683049147442325, + "loss": 1.6622, + "theoretical_loss": 4.40002823356844, + "tokens_seen": 184418304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768104312938816, + "loss": 1.6254, + "theoretical_loss": 4.39963216897044, + "tokens_seen": 184549376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047679037111334003, + "loss": 1.7437, + "theoretical_loss": 4.399236464267602, + "tokens_seen": 184680448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767703109327984, + "loss": 1.6554, + "theoretical_loss": 4.398841118877819, + "tokens_seen": 184811520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047675025075225676, + "loss": 1.6995, + "theoretical_loss": 4.398446132220338, + "tokens_seen": 184942592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767301905717151, + "loss": 1.9342, + "theoretical_loss": 4.398051503715753, + "tokens_seen": 185073664 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.5482649207115173, + "objective/train/avg_token_score": 0.00839152093976736, + "objective/train/docs_used": 115939, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.310957908630371, + "objective/train/original_loss": 4.36484956741333, + "objective/train/theoretical_loss": 4.397854323590102, + "objective/train/tokens_used": 185139200, + "objective/train/unlikelihood_loss": 0.10018301010131836, + "theoretical_loss": 4.397854323590102, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047671013039117353, + "loss": 1.7113, + "theoretical_loss": 4.397657232786008, + "tokens_seen": 185204736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047669007021063195, + "loss": 1.6987, + "theoretical_loss": 4.397263318854384, + "tokens_seen": 185335808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766700100300903, + "loss": 1.5346, + "theoretical_loss": 4.396869761345503, + "tokens_seen": 185466880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766499498495487, + "loss": 1.7575, + "theoretical_loss": 4.396476559685315, + "tokens_seen": 185597952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662988966900704, + "loss": 1.6408, + "theoretical_loss": 4.3960837133011035, + "tokens_seen": 185729024 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047660982948846545, + "loss": 1.8307, + "theoretical_loss": 4.395691221621476, + "tokens_seen": 185860096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765897693079238, + "loss": 1.8426, + "theoretical_loss": 4.39529908407636, + "tokens_seen": 185991168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765697091273822, + "loss": 1.6831, + "theoretical_loss": 4.394907300097002, + "tokens_seen": 186122240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654964894684054, + "loss": 1.6989, + "theoretical_loss": 4.39451586911596, + "tokens_seen": 186253312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765295887662989, + "loss": 1.7103, + "theoretical_loss": 4.394124790567101, + "tokens_seen": 186384384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765095285857573, + "loss": 1.7273, + "theoretical_loss": 4.393734063885599, + "tokens_seen": 186515456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764894684052157, + "loss": 1.6129, + "theoretical_loss": 4.39334368850793, + "tokens_seen": 186646528 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.47330155968666077, + "objective/train/avg_token_score": 0.007206100504845381, + "objective/train/docs_used": 116562, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.913322925567627, + "objective/train/original_loss": 4.169583797454834, + "objective/train/theoretical_loss": 4.392953663871862, + "objective/train/tokens_used": 186777600, + "objective/train/unlikelihood_loss": 0.11674590408802032, + "theoretical_loss": 4.392953663871862, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047646940822467404, + "loss": 1.7131, + "theoretical_loss": 4.392953663871862, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764493480441324, + "loss": 1.7657, + "theoretical_loss": 4.392563989416462, + "tokens_seen": 186908672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764292878635908, + "loss": 1.7479, + "theoretical_loss": 4.392174664582085, + "tokens_seen": 187039744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764092276830492, + "loss": 1.7493, + "theoretical_loss": 4.391785688810373, + "tokens_seen": 187170816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047638916750250755, + "loss": 1.7387, + "theoretical_loss": 4.391397061544247, + "tokens_seen": 187301888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763691073219659, + "loss": 1.6152, + "theoretical_loss": 4.3910087822279085, + "tokens_seen": 187432960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047634904714142427, + "loss": 1.5353, + "theoretical_loss": 4.390620850306832, + "tokens_seen": 187564032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763289869608827, + "loss": 1.5725, + "theoretical_loss": 4.390233265227764, + "tokens_seen": 187695104 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047630892678034105, + "loss": 1.631, + "theoretical_loss": 4.389846026438715, + "tokens_seen": 187826176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762888665997994, + "loss": 1.7898, + "theoretical_loss": 4.389459133388962, + "tokens_seen": 187957248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762688064192578, + "loss": 1.5287, + "theoretical_loss": 4.389072585529037, + "tokens_seen": 188088320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762487462387162, + "loss": 1.4302, + "theoretical_loss": 4.388686382310732, + "tokens_seen": 188219392 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047622868605817455, + "loss": 1.8187, + "theoretical_loss": 4.388300523187087, + "tokens_seen": 188350464 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.37365591526031494, + "objective/train/avg_token_score": 0.0153102558106184, + "objective/train/docs_used": 117858, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5269169807434082, + "objective/train/original_loss": 4.470926284790039, + "objective/train/theoretical_loss": 4.3881077224901786, + "objective/train/tokens_used": 188416000, + "objective/train/unlikelihood_loss": 0.11220584064722061, + "theoretical_loss": 4.3881077224901786, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762086258776329, + "loss": 1.5687, + "theoretical_loss": 4.38791500761239, + "tokens_seen": 188481536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761885656970913, + "loss": 1.5935, + "theoretical_loss": 4.3875298350421765, + "tokens_seen": 188612608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047616850551654964, + "loss": 1.7741, + "theoretical_loss": 4.387145004933218, + "tokens_seen": 188743680 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614844533600806, + "loss": 1.4192, + "theoretical_loss": 4.386760516743526, + "tokens_seen": 188874752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761283851554664, + "loss": 1.8067, + "theoretical_loss": 4.386376369932344, + "tokens_seen": 189005824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761083249749248, + "loss": 1.6904, + "theoretical_loss": 4.385992563960145, + "tokens_seen": 189136896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047608826479438314, + "loss": 1.7294, + "theoretical_loss": 4.385609098288628, + "tokens_seen": 189267968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047606820461384156, + "loss": 1.5139, + "theoretical_loss": 4.385225972380715, + "tokens_seen": 189399040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760481444332999, + "loss": 1.5383, + "theoretical_loss": 4.384843185700544, + "tokens_seen": 189530112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760280842527583, + "loss": 1.745, + "theoretical_loss": 4.384460737713471, + "tokens_seen": 189661184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047600802407221665, + "loss": 1.6476, + "theoretical_loss": 4.384078627886062, + "tokens_seen": 189792256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047598796389167506, + "loss": 1.7629, + "theoretical_loss": 4.38369685568609, + "tokens_seen": 189923328 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3760386109352112, + "objective/train/avg_token_score": 0.010193110443651676, + "objective/train/docs_used": 118609, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4706860780715942, + "objective/train/original_loss": 3.930041790008545, + "objective/train/theoretical_loss": 4.383315420582533, + "objective/train/tokens_used": 190054400, + "objective/train/unlikelihood_loss": 0.24754564464092255, + "theoretical_loss": 4.383315420582533, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759679037111334, + "loss": 1.862, + "theoretical_loss": 4.383315420582533, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759478435305918, + "loss": 1.5947, + "theoretical_loss": 4.38293432204557, + "tokens_seen": 190185472 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047592778335005015, + "loss": 1.7051, + "theoretical_loss": 4.3825535595465785, + "tokens_seen": 190316544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759077231695085, + "loss": 1.6088, + "theoretical_loss": 4.382173132558126, + "tokens_seen": 190447616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047588766298896693, + "loss": 1.6397, + "theoretical_loss": 4.381793040553973, + "tokens_seen": 190578688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758676028084253, + "loss": 1.662, + "theoretical_loss": 4.381413283009065, + "tokens_seen": 190709760 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047584754262788365, + "loss": 1.5988, + "theoretical_loss": 4.381033859399532, + "tokens_seen": 190840832 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475827482447342, + "loss": 1.7682, + "theoretical_loss": 4.380654769202683, + "tokens_seen": 190971904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047580742226680043, + "loss": 1.6793, + "theoretical_loss": 4.380276011897003, + "tokens_seen": 191102976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757873620862588, + "loss": 1.5176, + "theoretical_loss": 4.379897586962148, + "tokens_seen": 191234048 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047576730190571716, + "loss": 1.5289, + "theoretical_loss": 4.379519493878948, + "tokens_seen": 191365120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757472417251755, + "loss": 1.646, + "theoretical_loss": 4.379141732129394, + "tokens_seen": 191496192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757271815446339, + "loss": 1.6823, + "theoretical_loss": 4.378764301196642, + "tokens_seen": 191627264 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.46615347266197205, + "objective/train/avg_token_score": 0.02856089174747467, + "objective/train/docs_used": 119339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6883468627929688, + "objective/train/original_loss": 4.204484462738037, + "objective/train/theoretical_loss": 4.37857570962537, + "objective/train/tokens_used": 191692800, + "objective/train/unlikelihood_loss": 0.11059554666280746, + "theoretical_loss": 4.37857570962537, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757071213640923, + "loss": 1.699, + "theoretical_loss": 4.378387200565006, + "tokens_seen": 191758336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047568706118355066, + "loss": 1.8261, + "theoretical_loss": 4.378010429719957, + "tokens_seen": 191889408 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475667001003009, + "loss": 1.7145, + "theoretical_loss": 4.377633988148117, + "tokens_seen": 192020480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756469408224674, + "loss": 1.5378, + "theoretical_loss": 4.377257875337257, + "tokens_seen": 192151552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756268806419258, + "loss": 1.8218, + "theoretical_loss": 4.376882090776293, + "tokens_seen": 192282624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047560682046138416, + "loss": 1.7853, + "theoretical_loss": 4.376506633955286, + "tokens_seen": 192413696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755867602808425, + "loss": 1.4124, + "theoretical_loss": 4.37613150436543, + "tokens_seen": 192544768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755667001003009, + "loss": 1.6803, + "theoretical_loss": 4.375756701499063, + "tokens_seen": 192675840 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047554663991975925, + "loss": 1.5873, + "theoretical_loss": 4.375382224849648, + "tokens_seen": 192806912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047552657973921766, + "loss": 1.5583, + "theoretical_loss": 4.375008073911781, + "tokens_seen": 192937984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047550651955867603, + "loss": 1.6886, + "theoretical_loss": 4.374634248181182, + "tokens_seen": 193069056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754864593781344, + "loss": 1.6532, + "theoretical_loss": 4.374260747154692, + "tokens_seen": 193200128 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4188660681247711, + "objective/train/avg_token_score": 0.00520719401538372, + "objective/train/docs_used": 120055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6985862255096436, + "objective/train/original_loss": 4.125784873962402, + "objective/train/theoretical_loss": 4.373887570330275, + "objective/train/tokens_used": 193331200, + "objective/train/unlikelihood_loss": 0.16551171243190765, + "theoretical_loss": 4.373887570330275, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047546639919759275, + "loss": 1.6246, + "theoretical_loss": 4.373887570330275, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047544633901705117, + "loss": 1.4885, + "theoretical_loss": 4.373514717207009, + "tokens_seen": 193462272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047542627883650953, + "loss": 1.4034, + "theoretical_loss": 4.373142187285083, + "tokens_seen": 193593344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754062186559679, + "loss": 1.4975, + "theoretical_loss": 4.372769980065797, + "tokens_seen": 193724416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047538615847542625, + "loss": 1.6481, + "theoretical_loss": 4.372398095051559, + "tokens_seen": 193855488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753660982948846, + "loss": 1.4715, + "theoretical_loss": 4.372026531745877, + "tokens_seen": 193986560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047534603811434303, + "loss": 1.4546, + "theoretical_loss": 4.37165528965336, + "tokens_seen": 194117632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753259779338014, + "loss": 1.4513, + "theoretical_loss": 4.371284368279714, + "tokens_seen": 194248704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047530591775325976, + "loss": 1.7305, + "theoretical_loss": 4.3709137671317375, + "tokens_seen": 194379776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752858575727181, + "loss": 1.4355, + "theoretical_loss": 4.370543485717322, + "tokens_seen": 194510848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047526579739217654, + "loss": 1.5039, + "theoretical_loss": 4.370173523545443, + "tokens_seen": 194641920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752457372116349, + "loss": 1.6506, + "theoretical_loss": 4.369803880126162, + "tokens_seen": 194772992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047522567703109326, + "loss": 1.4202, + "theoretical_loss": 4.369434554970621, + "tokens_seen": 194904064 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.42467010021209717, + "objective/train/avg_token_score": 0.023077575489878654, + "objective/train/docs_used": 121265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6124404668807983, + "objective/train/original_loss": 3.863985300064087, + "objective/train/theoretical_loss": 4.369250011589279, + "objective/train/tokens_used": 194969600, + "objective/train/unlikelihood_loss": 0.3769527077674866, + "theoretical_loss": 4.369250011589279, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752056168505517, + "loss": 1.6853, + "theoretical_loss": 4.369065547591038, + "tokens_seen": 195035136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047518555667001004, + "loss": 1.7059, + "theoretical_loss": 4.36869685750071, + "tokens_seen": 195166208 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047516549648946846, + "loss": 1.6439, + "theoretical_loss": 4.368328484214002, + "tokens_seen": 195297280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751454363089268, + "loss": 1.5728, + "theoretical_loss": 4.3679604272463495, + "tokens_seen": 195428352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751253761283852, + "loss": 1.6146, + "theoretical_loss": 4.367592686114252, + "tokens_seen": 195559424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047510531594784354, + "loss": 1.563, + "theoretical_loss": 4.367225260335272, + "tokens_seen": 195690496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047508525576730196, + "loss": 1.6781, + "theoretical_loss": 4.366858149428032, + "tokens_seen": 195821568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750651955867603, + "loss": 1.5683, + "theoretical_loss": 4.366491352912211, + "tokens_seen": 195952640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750451354062187, + "loss": 1.7887, + "theoretical_loss": 4.366124870308541, + "tokens_seen": 196083712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047502507522567705, + "loss": 1.5592, + "theoretical_loss": 4.3657587011388035, + "tokens_seen": 196214784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047500501504513546, + "loss": 1.7372, + "theoretical_loss": 4.365392844925829, + "tokens_seen": 196345856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749849548645938, + "loss": 1.6439, + "theoretical_loss": 4.365027301193491, + "tokens_seen": 196476928 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4978005886077881, + "objective/train/avg_token_score": 0.0068794176913797855, + "objective/train/docs_used": 121711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8076094388961792, + "objective/train/original_loss": 4.219160556793213, + "objective/train/theoretical_loss": 4.364662069466704, + "objective/train/tokens_used": 196608000, + "objective/train/unlikelihood_loss": 0.12803588807582855, + "theoretical_loss": 4.364662069466704, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749648946840522, + "loss": 1.6509, + "theoretical_loss": 4.364662069466704, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494483450351055, + "loss": 1.4878, + "theoretical_loss": 4.364297149271423, + "tokens_seen": 196739072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749247743229689, + "loss": 1.5957, + "theoretical_loss": 4.363932540134638, + "tokens_seen": 196870144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047490471414242733, + "loss": 1.705, + "theoretical_loss": 4.363568241584368, + "tokens_seen": 197001216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748846539618857, + "loss": 1.5305, + "theoretical_loss": 4.363204253149667, + "tokens_seen": 197132288 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047486459378134405, + "loss": 1.7872, + "theoretical_loss": 4.362840574360612, + "tokens_seen": 197263360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748445336008024, + "loss": 1.7201, + "theoretical_loss": 4.362477204748305, + "tokens_seen": 197394432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047482447342026083, + "loss": 1.7059, + "theoretical_loss": 4.362114143844867, + "tokens_seen": 197525504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748044132397192, + "loss": 1.557, + "theoretical_loss": 4.3617513911834385, + "tokens_seen": 197656576 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478435305917756, + "loss": 1.6276, + "theoretical_loss": 4.3613889462981765, + "tokens_seen": 197787648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747642928786359, + "loss": 1.5583, + "theoretical_loss": 4.361026808724247, + "tokens_seen": 197918720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747442326980943, + "loss": 1.7629, + "theoretical_loss": 4.360664977997828, + "tokens_seen": 198049792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747241725175527, + "loss": 1.4869, + "theoretical_loss": 4.360303453656103, + "tokens_seen": 198180864 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3821480870246887, + "objective/train/avg_token_score": 0.010640091262757778, + "objective/train/docs_used": 122248, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6288254261016846, + "objective/train/original_loss": 4.3907952308654785, + "objective/train/theoretical_loss": 4.360122806235145, + "objective/train/tokens_used": 198246400, + "objective/train/unlikelihood_loss": 0.1243598461151123, + "theoretical_loss": 4.360122806235145, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470411233701106, + "loss": 1.6806, + "theoretical_loss": 4.359942235237257, + "tokens_seen": 198311936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746840521564694, + "loss": 1.6821, + "theoretical_loss": 4.359581322280479, + "tokens_seen": 198443008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746639919759278, + "loss": 1.7165, + "theoretical_loss": 4.359220714325954, + "tokens_seen": 198574080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746439317953862, + "loss": 1.6884, + "theoretical_loss": 4.358860410914861, + "tokens_seen": 198705152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047462387161484456, + "loss": 1.6764, + "theoretical_loss": 4.358500411589375, + "tokens_seen": 198836224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746038114343029, + "loss": 1.874, + "theoretical_loss": 4.358140715892658, + "tokens_seen": 198967296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745837512537613, + "loss": 1.5337, + "theoretical_loss": 4.357781323368857, + "tokens_seen": 199098368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047456369107321965, + "loss": 1.916, + "theoretical_loss": 4.357422233563106, + "tokens_seen": 199229440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454363089267807, + "loss": 1.5149, + "theoretical_loss": 4.35706344602152, + "tokens_seen": 199360512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047452357071213643, + "loss": 1.6638, + "theoretical_loss": 4.356704960291191, + "tokens_seen": 199491584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745035105315948, + "loss": 1.7224, + "theoretical_loss": 4.356346775920185, + "tokens_seen": 199622656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047448345035105315, + "loss": 1.6415, + "theoretical_loss": 4.355988892457546, + "tokens_seen": 199753728 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.5615224838256836, + "objective/train/avg_token_score": 0.008392857387661934, + "objective/train/docs_used": 123553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.331106185913086, + "objective/train/original_loss": 4.441834449768066, + "objective/train/theoretical_loss": 4.355631309453283, + "objective/train/tokens_used": 199884800, + "objective/train/unlikelihood_loss": 0.08705087006092072, + "theoretical_loss": 4.355631309453283, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047446339017051157, + "loss": 1.9122, + "theoretical_loss": 4.355631309453283, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444332998996993, + "loss": 1.8121, + "theoretical_loss": 4.355274026458375, + "tokens_seen": 200015872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744232698094283, + "loss": 1.7729, + "theoretical_loss": 4.354917043024765, + "tokens_seen": 200146944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047440320962888666, + "loss": 1.6087, + "theoretical_loss": 4.354560358705358, + "tokens_seen": 200278016 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474383149448345, + "loss": 1.4376, + "theoretical_loss": 4.35420397305402, + "tokens_seen": 200409088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436308926780343, + "loss": 1.8095, + "theoretical_loss": 4.353847885625571, + "tokens_seen": 200540160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743430290872618, + "loss": 1.672, + "theoretical_loss": 4.353492095975787, + "tokens_seen": 200671232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432296890672016, + "loss": 1.7417, + "theoretical_loss": 4.353136603661392, + "tokens_seen": 200802304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743029087261785, + "loss": 1.6302, + "theoretical_loss": 4.352781408240065, + "tokens_seen": 200933376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047428284854563694, + "loss": 1.7316, + "theoretical_loss": 4.352426509270425, + "tokens_seen": 201064448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742627883650953, + "loss": 1.8957, + "theoretical_loss": 4.352071906312037, + "tokens_seen": 201195520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047424272818455366, + "loss": 1.7722, + "theoretical_loss": 4.351717598925406, + "tokens_seen": 201326592 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474222668004012, + "loss": 1.6547, + "theoretical_loss": 4.351363586671976, + "tokens_seen": 201457664 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4540567100048065, + "objective/train/avg_token_score": 0.013070252723991871, + "objective/train/docs_used": 124253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9555007219314575, + "objective/train/original_loss": 4.570303916931152, + "objective/train/theoretical_loss": 4.351186691083417, + "objective/train/tokens_used": 201523200, + "objective/train/unlikelihood_loss": 0.11967580020427704, + "theoretical_loss": 4.351186691083417, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742026078234704, + "loss": 1.6592, + "theoretical_loss": 4.351009869114124, + "tokens_seen": 201588736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741825476429288, + "loss": 1.652, + "theoretical_loss": 4.350656445815164, + "tokens_seen": 201719808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416248746238716, + "loss": 1.6998, + "theoretical_loss": 4.350303316339337, + "tokens_seen": 201850880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741424272818455, + "loss": 1.637, + "theoretical_loss": 4.349950480251813, + "tokens_seen": 201981952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741223671013039, + "loss": 1.443, + "theoretical_loss": 4.349597937118687, + "tokens_seen": 202113024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741023069207623, + "loss": 1.719, + "theoretical_loss": 4.349245686506976, + "tokens_seen": 202244096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047408224674022067, + "loss": 1.6887, + "theoretical_loss": 4.348893727984619, + "tokens_seen": 202375168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047406218655967903, + "loss": 1.7137, + "theoretical_loss": 4.348542061120469, + "tokens_seen": 202506240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740421263791374, + "loss": 1.5756, + "theoretical_loss": 4.3481906854842975, + "tokens_seen": 202637312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047402206619859575, + "loss": 1.8621, + "theoretical_loss": 4.347839600646786, + "tokens_seen": 202768384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047400200601805417, + "loss": 1.6352, + "theoretical_loss": 4.347488806179528, + "tokens_seen": 202899456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047398194583751253, + "loss": 1.7664, + "theoretical_loss": 4.347138301655021, + "tokens_seen": 203030528 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4085410535335541, + "objective/train/avg_token_score": 0.01795566827058792, + "objective/train/docs_used": 125838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5744194984436035, + "objective/train/original_loss": 3.834841251373291, + "objective/train/theoretical_loss": 4.346788086646671, + "objective/train/tokens_used": 203161600, + "objective/train/unlikelihood_loss": 0.23488780856132507, + "theoretical_loss": 4.346788086646671, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739618856569709, + "loss": 1.4374, + "theoretical_loss": 4.346788086646671, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047394182547642926, + "loss": 1.6623, + "theoretical_loss": 4.346438160728785, + "tokens_seen": 203292672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739217652958877, + "loss": 1.8131, + "theoretical_loss": 4.346088523476569, + "tokens_seen": 203423744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047390170511534604, + "loss": 1.6644, + "theoretical_loss": 4.345739174466127, + "tokens_seen": 203554816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738816449348044, + "loss": 1.6942, + "theoretical_loss": 4.3453901132744575, + "tokens_seen": 203685888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047386158475426276, + "loss": 1.6128, + "theoretical_loss": 4.345041339479453, + "tokens_seen": 203816960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738415245737211, + "loss": 1.8607, + "theoretical_loss": 4.344692852659895, + "tokens_seen": 203948032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047382146439317954, + "loss": 1.6397, + "theoretical_loss": 4.344344652395451, + "tokens_seen": 204079104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738014042126379, + "loss": 1.5671, + "theoretical_loss": 4.343996738266677, + "tokens_seen": 204210176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047378134403209626, + "loss": 1.6248, + "theoretical_loss": 4.343649109855009, + "tokens_seen": 204341248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737612838515546, + "loss": 1.6748, + "theoretical_loss": 4.343301766742763, + "tokens_seen": 204472320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047374122367101304, + "loss": 1.6671, + "theoretical_loss": 4.342954708513136, + "tokens_seen": 204603392 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047372116349047146, + "loss": 1.5574, + "theoretical_loss": 4.3426079347501965, + "tokens_seen": 204734464 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3620479106903076, + "objective/train/avg_token_score": 0.014170818962156773, + "objective/train/docs_used": 126453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4098846912384033, + "objective/train/original_loss": 3.9272265434265137, + "objective/train/theoretical_loss": 4.342434654413995, + "objective/train/tokens_used": 204800000, + "objective/train/unlikelihood_loss": 0.20976464450359344, + "theoretical_loss": 4.342434654413995, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737011033099298, + "loss": 1.5437, + "theoretical_loss": 4.342261445038888, + "tokens_seen": 204865536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736810431293882, + "loss": 1.5255, + "theoretical_loss": 4.341915238965026, + "tokens_seen": 204996608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736609829488466, + "loss": 1.7084, + "theoretical_loss": 4.34156931611529, + "tokens_seen": 205127680 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047364092276830496, + "loss": 1.6739, + "theoretical_loss": 4.341223676077232, + "tokens_seen": 205258752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736208625877633, + "loss": 1.6936, + "theoretical_loss": 4.340878318439261, + "tokens_seen": 205389824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736008024072217, + "loss": 1.7022, + "theoretical_loss": 4.3405332427906504, + "tokens_seen": 205520896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047358074222668005, + "loss": 1.3979, + "theoretical_loss": 4.340188448721532, + "tokens_seen": 205651968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047356068204613847, + "loss": 1.7449, + "theoretical_loss": 4.339843935822895, + "tokens_seen": 205783040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047354062186559683, + "loss": 1.5221, + "theoretical_loss": 4.339499703686579, + "tokens_seen": 205914112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735205616850552, + "loss": 1.6496, + "theoretical_loss": 4.339155751905282, + "tokens_seen": 206045184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047350050150451355, + "loss": 1.6696, + "theoretical_loss": 4.338812080072545, + "tokens_seen": 206176256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047348044132397197, + "loss": 1.7524, + "theoretical_loss": 4.3384686877827585, + "tokens_seen": 206307328 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.37756597995758057, + "objective/train/avg_token_score": 0.015428772196173668, + "objective/train/docs_used": 127848, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4948502779006958, + "objective/train/original_loss": 4.056982040405273, + "objective/train/theoretical_loss": 4.33812557463116, + "objective/train/tokens_used": 206438400, + "objective/train/unlikelihood_loss": 0.15866130590438843, + "theoretical_loss": 4.33812557463116, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047346038114343033, + "loss": 1.5575, + "theoretical_loss": 4.33812557463116, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734403209628887, + "loss": 1.4098, + "theoretical_loss": 4.337782740213827, + "tokens_seen": 206569472 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047342026078234706, + "loss": 1.6327, + "theoretical_loss": 4.337440184127679, + "tokens_seen": 206700544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734002006018054, + "loss": 1.3983, + "theoretical_loss": 4.337097905970471, + "tokens_seen": 206831616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047338014042126383, + "loss": 1.7989, + "theoretical_loss": 4.336755905340797, + "tokens_seen": 206962688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733600802407222, + "loss": 1.6815, + "theoretical_loss": 4.336414181838082, + "tokens_seen": 207093760 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047334002006018056, + "loss": 1.6673, + "theoretical_loss": 4.336072735062583, + "tokens_seen": 207224832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733199598796389, + "loss": 1.7324, + "theoretical_loss": 4.335731564615387, + "tokens_seen": 207355904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047329989969909734, + "loss": 1.5223, + "theoretical_loss": 4.335390670098407, + "tokens_seen": 207486976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732798395185557, + "loss": 1.7556, + "theoretical_loss": 4.335050051114379, + "tokens_seen": 207618048 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325977933801406, + "loss": 1.6275, + "theoretical_loss": 4.334709707266865, + "tokens_seen": 207749120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732397191574724, + "loss": 1.7843, + "theoretical_loss": 4.334369638160242, + "tokens_seen": 207880192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732196589769308, + "loss": 1.6139, + "theoretical_loss": 4.334029843399709, + "tokens_seen": 208011264 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4079912006855011, + "objective/train/avg_token_score": 0.016537239775061607, + "objective/train/docs_used": 128338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6732548475265503, + "objective/train/original_loss": 4.391384124755859, + "objective/train/theoretical_loss": 4.333860048776074, + "objective/train/tokens_used": 208076800, + "objective/train/unlikelihood_loss": 0.11190986633300781, + "theoretical_loss": 4.333860048776074, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731995987963892, + "loss": 1.8005, + "theoretical_loss": 4.33369032259128, + "tokens_seen": 208142336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047317953861584757, + "loss": 1.4683, + "theoretical_loss": 4.33335107534178, + "tokens_seen": 208273408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047315947843530593, + "loss": 1.5492, + "theoretical_loss": 4.3330121012588485, + "tokens_seen": 208404480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731394182547643, + "loss": 1.6122, + "theoretical_loss": 4.332673399950932, + "tokens_seen": 208535552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731193580742227, + "loss": 1.6955, + "theoretical_loss": 4.332334971027284, + "tokens_seen": 208666624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047309929789368107, + "loss": 1.4314, + "theoretical_loss": 4.331996814097963, + "tokens_seen": 208797696 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047307923771313943, + "loss": 1.599, + "theoretical_loss": 4.331658928773831, + "tokens_seen": 208928768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730591775325978, + "loss": 1.7127, + "theoretical_loss": 4.33132131466655, + "tokens_seen": 209059840 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303911735205615, + "loss": 1.5623, + "theoretical_loss": 4.330983971388578, + "tokens_seen": 209190912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047301905717151457, + "loss": 1.4685, + "theoretical_loss": 4.330646898553173, + "tokens_seen": 209321984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047299899699097293, + "loss": 1.5697, + "theoretical_loss": 4.330310095774383, + "tokens_seen": 209453056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729789368104313, + "loss": 1.652, + "theoretical_loss": 4.329973562667053, + "tokens_seen": 209584128 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.45869991183280945, + "objective/train/avg_token_score": 0.025695886462926865, + "objective/train/docs_used": 129483, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.688183307647705, + "objective/train/original_loss": 4.105484485626221, + "objective/train/theoretical_loss": 4.329637298846812, + "objective/train/tokens_used": 209715200, + "objective/train/unlikelihood_loss": 0.14652951061725616, + "theoretical_loss": 4.329637298846812, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047295887662988966, + "loss": 1.675, + "theoretical_loss": 4.329637298846812, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729388164493481, + "loss": 1.5683, + "theoretical_loss": 4.32930130393008, + "tokens_seen": 209846272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047291875626880644, + "loss": 1.6511, + "theoretical_loss": 4.3289655775340625, + "tokens_seen": 209977344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728986960882648, + "loss": 1.5679, + "theoretical_loss": 4.328630119276747, + "tokens_seen": 210108416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047287863590772316, + "loss": 1.6666, + "theoretical_loss": 4.328294928776903, + "tokens_seen": 210239488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728585757271815, + "loss": 1.6358, + "theoretical_loss": 4.327960005654081, + "tokens_seen": 210370560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047283851554663994, + "loss": 1.5076, + "theoretical_loss": 4.327625349528605, + "tokens_seen": 210501632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728184553660983, + "loss": 1.5038, + "theoretical_loss": 4.3272909600215765, + "tokens_seen": 210632704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047279839518555666, + "loss": 1.7096, + "theoretical_loss": 4.326956836754871, + "tokens_seen": 210763776 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472778335005015, + "loss": 1.4226, + "theoretical_loss": 4.326622979351132, + "tokens_seen": 210894848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047275827482447344, + "loss": 1.5037, + "theoretical_loss": 4.326289387433776, + "tokens_seen": 211025920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727382146439318, + "loss": 1.7344, + "theoretical_loss": 4.325956060626982, + "tokens_seen": 211156992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047271815446339017, + "loss": 1.7053, + "theoretical_loss": 4.325622998555697, + "tokens_seen": 211288064 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.433956503868103, + "objective/train/avg_token_score": 0.010776055976748466, + "objective/train/docs_used": 130263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.608245849609375, + "objective/train/original_loss": 4.053385257720947, + "objective/train/theoretical_loss": 4.3254565666788745, + "objective/train/tokens_used": 211353600, + "objective/train/unlikelihood_loss": 0.1744779348373413, + "theoretical_loss": 4.3254565666788745, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047269809428284853, + "loss": 1.6731, + "theoretical_loss": 4.325290200845629, + "tokens_seen": 211419136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047267803410230695, + "loss": 1.7337, + "theoretical_loss": 4.324957667123249, + "tokens_seen": 211550208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726579739217653, + "loss": 1.7319, + "theoretical_loss": 4.324625397015783, + "tokens_seen": 211681280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047263791374122367, + "loss": 1.6882, + "theoretical_loss": 4.324293390151218, + "tokens_seen": 211812352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047261785356068203, + "loss": 1.5198, + "theoretical_loss": 4.323961646158294, + "tokens_seen": 211943424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725977933801404, + "loss": 1.5977, + "theoretical_loss": 4.323630164666502, + "tokens_seen": 212074496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725777331995988, + "loss": 1.6756, + "theoretical_loss": 4.323298945306089, + "tokens_seen": 212205568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725576730190572, + "loss": 1.6358, + "theoretical_loss": 4.322967987708043, + "tokens_seen": 212336640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047253761283851554, + "loss": 1.6253, + "theoretical_loss": 4.322637291504106, + "tokens_seen": 212467712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725175526579739, + "loss": 1.5415, + "theoretical_loss": 4.322306856326761, + "tokens_seen": 212598784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724974924774323, + "loss": 1.7128, + "theoretical_loss": 4.321976681809236, + "tokens_seen": 212729856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724774322968907, + "loss": 1.658, + "theoretical_loss": 4.321646767585497, + "tokens_seen": 212860928 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.32704055309295654, + "objective/train/avg_token_score": 0.014311397448182106, + "objective/train/docs_used": 131630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.350407600402832, + "objective/train/original_loss": 4.261425971984863, + "objective/train/theoretical_loss": 4.321317113290252, + "objective/train/tokens_used": 212992000, + "objective/train/unlikelihood_loss": 0.19203782081604004, + "theoretical_loss": 4.321317113290252, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047245737211634904, + "loss": 1.4489, + "theoretical_loss": 4.321317113290252, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724373119358074, + "loss": 1.6253, + "theoretical_loss": 4.320987718558945, + "tokens_seen": 213123072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047241725175526576, + "loss": 1.6789, + "theoretical_loss": 4.320658583027755, + "tokens_seen": 213254144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723971915747242, + "loss": 1.4694, + "theoretical_loss": 4.3203297063335935, + "tokens_seen": 213385216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047237713139418254, + "loss": 1.515, + "theoretical_loss": 4.320001088114105, + "tokens_seen": 213516288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723570712136409, + "loss": 1.7089, + "theoretical_loss": 4.31967272800766, + "tokens_seen": 213647360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047233701103309927, + "loss": 1.527, + "theoretical_loss": 4.319344625653361, + "tokens_seen": 213778432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723169508525577, + "loss": 1.7138, + "theoretical_loss": 4.319016780691033, + "tokens_seen": 213909504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047229689067201605, + "loss": 1.7267, + "theoretical_loss": 4.318689192761225, + "tokens_seen": 214040576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722768304914744, + "loss": 1.6687, + "theoretical_loss": 4.318361861505207, + "tokens_seen": 214171648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047225677031093277, + "loss": 1.6014, + "theoretical_loss": 4.318034786564971, + "tokens_seen": 214302720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047223671013039113, + "loss": 1.5851, + "theoretical_loss": 4.3177079675832255, + "tokens_seen": 214433792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722166499498496, + "loss": 1.3161, + "theoretical_loss": 4.3173814042033944, + "tokens_seen": 214564864 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4389662742614746, + "objective/train/avg_token_score": 0.023373806849122047, + "objective/train/docs_used": 132228, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6921892166137695, + "objective/train/original_loss": 4.019752502441406, + "objective/train/theoretical_loss": 4.317218218252963, + "objective/train/tokens_used": 214630400, + "objective/train/unlikelihood_loss": 0.20437592267990112, + "theoretical_loss": 4.317218218252963, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047219658976930797, + "loss": 1.6002, + "theoretical_loss": 4.317055096069618, + "tokens_seen": 214695936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047217652958876633, + "loss": 1.6065, + "theoretical_loss": 4.316729042826745, + "tokens_seen": 214827008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721564694082247, + "loss": 1.7483, + "theoretical_loss": 4.316403244120339, + "tokens_seen": 214958080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721364092276831, + "loss": 1.631, + "theoretical_loss": 4.316077699596671, + "tokens_seen": 215089152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047211634904714147, + "loss": 1.8237, + "theoretical_loss": 4.315752408902716, + "tokens_seen": 215220224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047209628886659983, + "loss": 1.6567, + "theoretical_loss": 4.315427371686157, + "tokens_seen": 215351296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720762286860582, + "loss": 1.6742, + "theoretical_loss": 4.315102587595379, + "tokens_seen": 215482368 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047205616850551656, + "loss": 1.7232, + "theoretical_loss": 4.314778056279468, + "tokens_seen": 215613440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047203610832497497, + "loss": 1.5351, + "theoretical_loss": 4.314453777388209, + "tokens_seen": 215744512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047201604814443333, + "loss": 1.5639, + "theoretical_loss": 4.314129750572087, + "tokens_seen": 215875584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719959879638917, + "loss": 1.4961, + "theoretical_loss": 4.313805975482278, + "tokens_seen": 216006656 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047197592778335006, + "loss": 1.7063, + "theoretical_loss": 4.313482451770659, + "tokens_seen": 216137728 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.47482892870903015, + "objective/train/avg_token_score": 0.027175776660442352, + "objective/train/docs_used": 133529, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9989675283432007, + "objective/train/original_loss": 4.329239845275879, + "objective/train/theoretical_loss": 4.3131591790897925, + "objective/train/tokens_used": 216268800, + "objective/train/unlikelihood_loss": 0.155557319521904, + "theoretical_loss": 4.3131591790897925, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719558676028085, + "loss": 1.4553, + "theoretical_loss": 4.3131591790897925, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193580742226684, + "loss": 1.6868, + "theoretical_loss": 4.312836157092934, + "tokens_seen": 216399872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719157472417252, + "loss": 1.4115, + "theoretical_loss": 4.31251338543403, + "tokens_seen": 216530944 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047189568706118356, + "loss": 1.6705, + "theoretical_loss": 4.312190863767708, + "tokens_seen": 216662016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718756268806419, + "loss": 1.6034, + "theoretical_loss": 4.311868591749287, + "tokens_seen": 216793088 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047185556670010034, + "loss": 1.7697, + "theoretical_loss": 4.311546569034767, + "tokens_seen": 216924160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718355065195587, + "loss": 1.6095, + "theoretical_loss": 4.311224795280825, + "tokens_seen": 217055232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047181544633901706, + "loss": 1.7137, + "theoretical_loss": 4.310903270144825, + "tokens_seen": 217186304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717953861584754, + "loss": 1.5225, + "theoretical_loss": 4.310581993284805, + "tokens_seen": 217317376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177532597793384, + "loss": 1.8289, + "theoretical_loss": 4.310260964359479, + "tokens_seen": 217448448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717552657973922, + "loss": 1.6959, + "theoretical_loss": 4.309940183028236, + "tokens_seen": 217579520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173520561685057, + "loss": 1.604, + "theoretical_loss": 4.309619648951139, + "tokens_seen": 217710592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047171514543630893, + "loss": 1.6158, + "theoretical_loss": 4.30929936178892, + "tokens_seen": 217841664 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.5201612710952759, + "objective/train/avg_token_score": 0.025473540648818016, + "objective/train/docs_used": 134228, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.066497325897217, + "objective/train/original_loss": 3.987381935119629, + "objective/train/theoretical_loss": 4.309139310695053, + "objective/train/tokens_used": 217907200, + "objective/train/unlikelihood_loss": 0.17281749844551086, + "theoretical_loss": 4.309139310695053, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169508525576735, + "loss": 1.7519, + "theoretical_loss": 4.308979321202983, + "tokens_seen": 217972736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716750250752257, + "loss": 1.7713, + "theoretical_loss": 4.308659526855396, + "tokens_seen": 218103808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047165496489468407, + "loss": 1.6655, + "theoretical_loss": 4.308339978408897, + "tokens_seen": 218234880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163490471414243, + "loss": 1.7458, + "theoretical_loss": 4.308020675526883, + "tokens_seen": 218365952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716148445336008, + "loss": 1.6181, + "theoretical_loss": 4.30770161787342, + "tokens_seen": 218497024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715947843530592, + "loss": 1.5035, + "theoretical_loss": 4.307382805113228, + "tokens_seen": 218628096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715747241725176, + "loss": 1.616, + "theoretical_loss": 4.307064236911692, + "tokens_seen": 218759168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047155466399197594, + "loss": 1.806, + "theoretical_loss": 4.306745912934849, + "tokens_seen": 218890240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715346038114343, + "loss": 1.6041, + "theoretical_loss": 4.306427832849394, + "tokens_seen": 219021312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715145436308927, + "loss": 1.8418, + "theoretical_loss": 4.306109996322679, + "tokens_seen": 219152384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714944834503511, + "loss": 1.5547, + "theoretical_loss": 4.305792403022703, + "tokens_seen": 219283456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047147442326980944, + "loss": 1.2979, + "theoretical_loss": 4.305475052618119, + "tokens_seen": 219414528 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4022482931613922, + "objective/train/avg_token_score": 0.006948302499949932, + "objective/train/docs_used": 135676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5557243824005127, + "objective/train/original_loss": 3.7284510135650635, + "objective/train/theoretical_loss": 4.305157944778228, + "objective/train/tokens_used": 219545600, + "objective/train/unlikelihood_loss": 0.15928247570991516, + "theoretical_loss": 4.305157944778228, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714543630892678, + "loss": 1.86, + "theoretical_loss": 4.305157944778228, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047143430290872616, + "loss": 1.7673, + "theoretical_loss": 4.304841079172979, + "tokens_seen": 219676672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714142427281846, + "loss": 1.6245, + "theoretical_loss": 4.304524455472965, + "tokens_seen": 219807744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139418254764294, + "loss": 1.8468, + "theoretical_loss": 4.304208073349426, + "tokens_seen": 219938816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713741223671013, + "loss": 1.5617, + "theoretical_loss": 4.30389193247424, + "tokens_seen": 220069888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135406218655967, + "loss": 1.7598, + "theoretical_loss": 4.303576032519931, + "tokens_seen": 220200960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713340020060181, + "loss": 1.7091, + "theoretical_loss": 4.303260373159659, + "tokens_seen": 220332032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047131394182547645, + "loss": 1.7758, + "theoretical_loss": 4.30294495406722, + "tokens_seen": 220463104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712938816449348, + "loss": 1.6308, + "theoretical_loss": 4.302629774917049, + "tokens_seen": 220594176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047127382146439317, + "loss": 1.6361, + "theoretical_loss": 4.302314835384214, + "tokens_seen": 220725248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125376128385153, + "loss": 1.6959, + "theoretical_loss": 4.302000135144416, + "tokens_seen": 220856320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047123370110330995, + "loss": 1.7791, + "theoretical_loss": 4.301685673873987, + "tokens_seen": 220987392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712136409227683, + "loss": 1.6592, + "theoretical_loss": 4.301371451249888, + "tokens_seen": 221118464 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4833821952342987, + "objective/train/avg_token_score": 0.009588075801730156, + "objective/train/docs_used": 136371, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.763645887374878, + "objective/train/original_loss": 3.7894089221954346, + "objective/train/theoretical_loss": 4.301214429329433, + "objective/train/tokens_used": 221184000, + "objective/train/unlikelihood_loss": 0.19660082459449768, + "theoretical_loss": 4.301214429329433, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711935807422267, + "loss": 1.6616, + "theoretical_loss": 4.301057466949707, + "tokens_seen": 221249536 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047117352056168504, + "loss": 1.7754, + "theoretical_loss": 4.30074372065166, + "tokens_seen": 221380608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115346038114345, + "loss": 1.5822, + "theoretical_loss": 4.300430212034587, + "tokens_seen": 221511680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711334002006018, + "loss": 1.7638, + "theoretical_loss": 4.300116940777951, + "tokens_seen": 221642752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711133400200602, + "loss": 1.8272, + "theoretical_loss": 4.299803906561835, + "tokens_seen": 221773824 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047109327983951854, + "loss": 1.6185, + "theoretical_loss": 4.299491109066947, + "tokens_seen": 221904896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710732196589769, + "loss": 1.6955, + "theoretical_loss": 4.2991785479746065, + "tokens_seen": 222035968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710531594784353, + "loss": 1.7621, + "theoretical_loss": 4.298866222966755, + "tokens_seen": 222167040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710330992978937, + "loss": 1.5053, + "theoretical_loss": 4.298554133725946, + "tokens_seen": 222298112 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047101303911735204, + "loss": 1.7638, + "theoretical_loss": 4.298242279935349, + "tokens_seen": 222429184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709929789368104, + "loss": 1.7638, + "theoretical_loss": 4.297930661278745, + "tokens_seen": 222560256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709729187562688, + "loss": 1.6024, + "theoretical_loss": 4.297619277440523, + "tokens_seen": 222691328 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4119012653827667, + "objective/train/avg_token_score": 0.012975508347153664, + "objective/train/docs_used": 137650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5277689695358276, + "objective/train/original_loss": 4.178741931915283, + "objective/train/theoretical_loss": 4.297308128105687, + "objective/train/tokens_used": 222822400, + "objective/train/unlikelihood_loss": 0.15160509943962097, + "theoretical_loss": 4.297308128105687, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709528585757272, + "loss": 1.4614, + "theoretical_loss": 4.297308128105687, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047093279839518555, + "loss": 1.6429, + "theoretical_loss": 4.296997212959842, + "tokens_seen": 222953472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709127382146439, + "loss": 1.6713, + "theoretical_loss": 4.296686531689204, + "tokens_seen": 223084544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047089267803410227, + "loss": 1.7281, + "theoretical_loss": 4.296376083980589, + "tokens_seen": 223215616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708726178535607, + "loss": 1.7329, + "theoretical_loss": 4.296065869521421, + "tokens_seen": 223346688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047085255767301905, + "loss": 1.6212, + "theoretical_loss": 4.2957558879997215, + "tokens_seen": 223477760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708324974924774, + "loss": 1.7222, + "theoretical_loss": 4.295446139104112, + "tokens_seen": 223608832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708124373119358, + "loss": 1.5936, + "theoretical_loss": 4.295136622523817, + "tokens_seen": 223739904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707923771313942, + "loss": 1.5582, + "theoretical_loss": 4.294827337948651, + "tokens_seen": 223870976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047077231695085255, + "loss": 1.621, + "theoretical_loss": 4.29451828506903, + "tokens_seen": 224002048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707522567703109, + "loss": 1.7204, + "theoretical_loss": 4.29420946357596, + "tokens_seen": 224133120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707321965897693, + "loss": 1.6165, + "theoretical_loss": 4.293900873161043, + "tokens_seen": 224264192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047071213640922775, + "loss": 1.6774, + "theoretical_loss": 4.293592513516469, + "tokens_seen": 224395264 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.3668743968009949, + "objective/train/avg_token_score": 0.00708651402965188, + "objective/train/docs_used": 138458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4711482524871826, + "objective/train/original_loss": 4.133864879608154, + "objective/train/theoretical_loss": 4.293438420137031, + "objective/train/tokens_used": 224460800, + "objective/train/unlikelihood_loss": 0.21583233773708344, + "theoretical_loss": 4.293438420137031, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706920762286861, + "loss": 1.4837, + "theoretical_loss": 4.293284384335017, + "tokens_seen": 224526336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047067201604814447, + "loss": 1.646, + "theoretical_loss": 4.292976485310057, + "tokens_seen": 224657408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047065195586760283, + "loss": 1.4897, + "theoretical_loss": 4.292668816135545, + "tokens_seen": 224788480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706318956870612, + "loss": 1.6201, + "theoretical_loss": 4.29236137650602, + "tokens_seen": 224919552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706118355065196, + "loss": 1.7653, + "theoretical_loss": 4.292054166116605, + "tokens_seen": 225050624 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470591775325978, + "loss": 1.5181, + "theoretical_loss": 4.2917471846630075, + "tokens_seen": 225181696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047057171514543634, + "loss": 1.5244, + "theoretical_loss": 4.291440431841513, + "tokens_seen": 225312768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705516549648947, + "loss": 1.3208, + "theoretical_loss": 4.291133907348989, + "tokens_seen": 225443840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705315947843531, + "loss": 1.6231, + "theoretical_loss": 4.29082761088288, + "tokens_seen": 225574912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705115346038115, + "loss": 1.4922, + "theoretical_loss": 4.290521542141203, + "tokens_seen": 225705984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047049147442326984, + "loss": 1.63, + "theoretical_loss": 4.290215700822556, + "tokens_seen": 225837056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704714142427282, + "loss": 1.6671, + "theoretical_loss": 4.289910086626108, + "tokens_seen": 225968128 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.519305944442749, + "objective/train/avg_token_score": 0.011661575175821781, + "objective/train/docs_used": 139641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.168332815170288, + "objective/train/original_loss": 4.076672077178955, + "objective/train/theoretical_loss": 4.2896046992515995, + "objective/train/tokens_used": 226099200, + "objective/train/unlikelihood_loss": 0.17042392492294312, + "theoretical_loss": 4.2896046992515995, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047045135406218656, + "loss": 1.5969, + "theoretical_loss": 4.2896046992515995, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470431293881645, + "loss": 1.5568, + "theoretical_loss": 4.289299538399341, + "tokens_seen": 226230272 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047041123370110334, + "loss": 1.654, + "theoretical_loss": 4.288994603770215, + "tokens_seen": 226361344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703911735205617, + "loss": 1.651, + "theoretical_loss": 4.28868989506567, + "tokens_seen": 226492416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047037111334002007, + "loss": 1.8949, + "theoretical_loss": 4.288385411987722, + "tokens_seen": 226623488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703510531594785, + "loss": 1.5494, + "theoretical_loss": 4.2880811542389505, + "tokens_seen": 226754560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047033099297893685, + "loss": 1.4705, + "theoretical_loss": 4.287777121522501, + "tokens_seen": 226885632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703109327983952, + "loss": 1.6097, + "theoretical_loss": 4.287473313542077, + "tokens_seen": 227016704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047029087261785357, + "loss": 1.7498, + "theoretical_loss": 4.287169730001949, + "tokens_seen": 227147776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047027081243731193, + "loss": 1.7013, + "theoretical_loss": 4.286866370606943, + "tokens_seen": 227278848 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047025075225677035, + "loss": 1.8046, + "theoretical_loss": 4.286563235062444, + "tokens_seen": 227409920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702306920762287, + "loss": 1.6791, + "theoretical_loss": 4.286260323074394, + "tokens_seen": 227540992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702106318956871, + "loss": 1.578, + "theoretical_loss": 4.285957634349289, + "tokens_seen": 227672064 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.46957477927207947, + "objective/train/avg_token_score": 0.018808742985129356, + "objective/train/docs_used": 140376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8826521635055542, + "objective/train/original_loss": 4.291516304016113, + "objective/train/theoretical_loss": 4.285806373618774, + "objective/train/tokens_used": 227737600, + "objective/train/unlikelihood_loss": 0.15742744505405426, + "theoretical_loss": 4.285806373618774, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047019057171514544, + "loss": 1.8432, + "theoretical_loss": 4.285655168594182, + "tokens_seen": 227803136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047017051153460385, + "loss": 1.539, + "theoretical_loss": 4.285352925516676, + "tokens_seen": 227934208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701504513540622, + "loss": 1.6382, + "theoretical_loss": 4.285050904824925, + "tokens_seen": 228065280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701303911735206, + "loss": 1.5511, + "theoretical_loss": 4.284749106227636, + "tokens_seen": 228196352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047011033099297894, + "loss": 1.6545, + "theoretical_loss": 4.284447529434061, + "tokens_seen": 228327424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700902708124373, + "loss": 1.6648, + "theoretical_loss": 4.284146174154003, + "tokens_seen": 228458496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700702106318957, + "loss": 1.5866, + "theoretical_loss": 4.283845040097807, + "tokens_seen": 228589568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700501504513541, + "loss": 1.6361, + "theoretical_loss": 4.2835441269763646, + "tokens_seen": 228720640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047003009027081244, + "loss": 1.6915, + "theoretical_loss": 4.283243434501112, + "tokens_seen": 228851712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700100300902708, + "loss": 1.6013, + "theoretical_loss": 4.282942962384023, + "tokens_seen": 228982784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699899699097292, + "loss": 1.6088, + "theoretical_loss": 4.282642710337618, + "tokens_seen": 229113856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699699097291876, + "loss": 1.7422, + "theoretical_loss": 4.282342678074951, + "tokens_seen": 229244928 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.34762951731681824, + "objective/train/avg_token_score": 0.020571375265717506, + "objective/train/docs_used": 141693, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.228954553604126, + "objective/train/original_loss": 4.2740325927734375, + "objective/train/theoretical_loss": 4.282042865309616, + "objective/train/tokens_used": 229376000, + "objective/train/unlikelihood_loss": 0.13970565795898438, + "theoretical_loss": 4.282042865309616, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046994984954864595, + "loss": 1.5481, + "theoretical_loss": 4.282042865309616, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699297893681043, + "loss": 1.7614, + "theoretical_loss": 4.2817432717557455, + "tokens_seen": 229507072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046990972918756267, + "loss": 1.4883, + "theoretical_loss": 4.281443897128004, + "tokens_seen": 229638144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698896690070211, + "loss": 1.5377, + "theoretical_loss": 4.281144741141593, + "tokens_seen": 229769216 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046986960882647945, + "loss": 1.6032, + "theoretical_loss": 4.2808458035122445, + "tokens_seen": 229900288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698495486459378, + "loss": 1.6431, + "theoretical_loss": 4.280547083956224, + "tokens_seen": 230031360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698294884653962, + "loss": 1.7657, + "theoretical_loss": 4.280248582190324, + "tokens_seen": 230162432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698094282848546, + "loss": 1.6122, + "theoretical_loss": 4.279950297931869, + "tokens_seen": 230293504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046978936810431295, + "loss": 1.5755, + "theoretical_loss": 4.279652230898709, + "tokens_seen": 230424576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697693079237713, + "loss": 1.557, + "theoretical_loss": 4.27935438080922, + "tokens_seen": 230555648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697492477432297, + "loss": 1.722, + "theoretical_loss": 4.279056747382306, + "tokens_seen": 230686720 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046972918756268804, + "loss": 1.5124, + "theoretical_loss": 4.27875933033739, + "tokens_seen": 230817792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046970912738214646, + "loss": 1.7483, + "theoretical_loss": 4.278462129394423, + "tokens_seen": 230948864 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.44086021184921265, + "objective/train/avg_token_score": 0.02034401148557663, + "objective/train/docs_used": 142407, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9175533056259155, + "objective/train/original_loss": 4.461835861206055, + "objective/train/theoretical_loss": 4.278313609873795, + "objective/train/tokens_used": 231014400, + "objective/train/unlikelihood_loss": 0.13161636888980865, + "theoretical_loss": 4.278313609873795, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696890672016048, + "loss": 1.6861, + "theoretical_loss": 4.278165144273871, + "tokens_seen": 231079936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696690070210632, + "loss": 1.6634, + "theoretical_loss": 4.277868374696725, + "tokens_seen": 231211008 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046964894684052154, + "loss": 1.7192, + "theoretical_loss": 4.277571820384491, + "tokens_seen": 231342080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046962888665997996, + "loss": 1.5473, + "theoretical_loss": 4.277275481059195, + "tokens_seen": 231473152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696088264794383, + "loss": 1.6103, + "theoretical_loss": 4.276979356443377, + "tokens_seen": 231604224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695887662988967, + "loss": 1.6004, + "theoretical_loss": 4.276683446260093, + "tokens_seen": 231735296 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046956870611835505, + "loss": 1.5718, + "theoretical_loss": 4.276387750232913, + "tokens_seen": 231866368 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046954864593781346, + "loss": 1.4526, + "theoretical_loss": 4.276092268085918, + "tokens_seen": 231997440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695285857572718, + "loss": 1.7252, + "theoretical_loss": 4.275796999543703, + "tokens_seen": 232128512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695085255767302, + "loss": 1.6623, + "theoretical_loss": 4.275501944331367, + "tokens_seen": 232259584 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046948846539618855, + "loss": 1.6335, + "theoretical_loss": 4.275207102174525, + "tokens_seen": 232390656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694684052156469, + "loss": 1.9984, + "theoretical_loss": 4.2749124727992935, + "tokens_seen": 232521728 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.44855815172195435, + "objective/train/avg_token_score": 0.007418564986437559, + "objective/train/docs_used": 143093, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7657521963119507, + "objective/train/original_loss": 3.696604013442993, + "objective/train/theoretical_loss": 4.274618055932298, + "objective/train/tokens_used": 232652800, + "objective/train/unlikelihood_loss": 0.1868204027414322, + "theoretical_loss": 4.274618055932298, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694483450351053, + "loss": 1.7135, + "theoretical_loss": 4.274618055932298, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694282848545637, + "loss": 1.6273, + "theoretical_loss": 4.27432385130067, + "tokens_seen": 232783872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046940822467402205, + "loss": 1.6464, + "theoretical_loss": 4.2740298586320415, + "tokens_seen": 232914944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693881644934804, + "loss": 1.6506, + "theoretical_loss": 4.27373607765455, + "tokens_seen": 233046016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046936810431293883, + "loss": 1.5552, + "theoretical_loss": 4.273442508096833, + "tokens_seen": 233177088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693480441323972, + "loss": 1.6676, + "theoretical_loss": 4.273149149688028, + "tokens_seen": 233308160 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046932798395185555, + "loss": 1.9249, + "theoretical_loss": 4.272856002157772, + "tokens_seen": 233439232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693079237713139, + "loss": 1.7331, + "theoretical_loss": 4.2725630652362, + "tokens_seen": 233570304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692878635907723, + "loss": 1.6774, + "theoretical_loss": 4.272270338653942, + "tokens_seen": 233701376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692678034102307, + "loss": 1.5886, + "theoretical_loss": 4.271977822142125, + "tokens_seen": 233832448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046924774322968906, + "loss": 1.6229, + "theoretical_loss": 4.27168551543237, + "tokens_seen": 233963520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692276830491474, + "loss": 1.6078, + "theoretical_loss": 4.27139341825679, + "tokens_seen": 234094592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046920762286860584, + "loss": 1.672, + "theoretical_loss": 4.27110153034799, + "tokens_seen": 234225664 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4761730134487152, + "objective/train/avg_token_score": 0.01180332899093628, + "objective/train/docs_used": 144323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8356056213378906, + "objective/train/original_loss": 4.213988780975342, + "objective/train/theoretical_loss": 4.270955664785207, + "objective/train/tokens_used": 234291200, + "objective/train/unlikelihood_loss": 0.13796842098236084, + "theoretical_loss": 4.270955664785207, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046918756268806425, + "loss": 1.6133, + "theoretical_loss": 4.2708098514390676, + "tokens_seen": 234356736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691675025075226, + "loss": 1.7033, + "theoretical_loss": 4.2705183812636065, + "tokens_seen": 234487808 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469147442326981, + "loss": 1.7059, + "theoretical_loss": 4.270227119555681, + "tokens_seen": 234618880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046912738214643934, + "loss": 1.7052, + "theoretical_loss": 4.269936066049852, + "tokens_seen": 234749952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691073219658977, + "loss": 1.4875, + "theoretical_loss": 4.269645220481166, + "tokens_seen": 234881024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690872617853561, + "loss": 1.7967, + "theoretical_loss": 4.269354582585156, + "tokens_seen": 235012096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690672016048145, + "loss": 1.5462, + "theoretical_loss": 4.269064152097835, + "tokens_seen": 235143168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046904714142427284, + "loss": 1.642, + "theoretical_loss": 4.268773928755701, + "tokens_seen": 235274240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690270812437312, + "loss": 1.7563, + "theoretical_loss": 4.268483912295735, + "tokens_seen": 235405312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690070210631896, + "loss": 1.7788, + "theoretical_loss": 4.268194102455395, + "tokens_seen": 235536384 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468986960882648, + "loss": 1.758, + "theoretical_loss": 4.267904498972618, + "tokens_seen": 235667456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046896690070210635, + "loss": 1.5842, + "theoretical_loss": 4.267615101585821, + "tokens_seen": 235798528 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.48765885829925537, + "objective/train/avg_token_score": 0.008590947836637497, + "objective/train/docs_used": 145084, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9980686902999878, + "objective/train/original_loss": 4.121566295623779, + "objective/train/theoretical_loss": 4.267325910033897, + "objective/train/tokens_used": 235929600, + "objective/train/unlikelihood_loss": 0.1698838621377945, + "theoretical_loss": 4.267325910033897, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689468405215647, + "loss": 1.6593, + "theoretical_loss": 4.267325910033897, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892678034102307, + "loss": 1.6164, + "theoretical_loss": 4.267036924056215, + "tokens_seen": 236060672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689067201604815, + "loss": 1.5894, + "theoretical_loss": 4.266748143392617, + "tokens_seen": 236191744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046888665997993985, + "loss": 1.6229, + "theoretical_loss": 4.26645956778342, + "tokens_seen": 236322816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688665997993982, + "loss": 1.805, + "theoretical_loss": 4.266171196969412, + "tokens_seen": 236453888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688465396188566, + "loss": 1.7046, + "theoretical_loss": 4.265883030691853, + "tokens_seen": 236584960 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468826479438315, + "loss": 1.6652, + "theoretical_loss": 4.265595068692473, + "tokens_seen": 236716032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046880641925777335, + "loss": 1.7922, + "theoretical_loss": 4.265307310713471, + "tokens_seen": 236847104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687863590772317, + "loss": 1.5168, + "theoretical_loss": 4.265019756497512, + "tokens_seen": 236978176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687662988966901, + "loss": 1.7535, + "theoretical_loss": 4.264732405787731, + "tokens_seen": 237109248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046874623871614844, + "loss": 1.6919, + "theoretical_loss": 4.264445258327724, + "tokens_seen": 237240320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872617853560686, + "loss": 1.7658, + "theoretical_loss": 4.264158313861557, + "tokens_seen": 237371392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687061183550652, + "loss": 1.6452, + "theoretical_loss": 4.2638715721337554, + "tokens_seen": 237502464 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4448924660682678, + "objective/train/avg_token_score": 0.019487597048282623, + "objective/train/docs_used": 146518, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6499921083450317, + "objective/train/original_loss": 4.144484519958496, + "objective/train/theoretical_loss": 4.263728277217032, + "objective/train/tokens_used": 237568000, + "objective/train/unlikelihood_loss": 0.14790794253349304, + "theoretical_loss": 4.263728277217032, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686860581745236, + "loss": 1.6665, + "theoretical_loss": 4.263585032889306, + "tokens_seen": 237633536 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046866599799398194, + "loss": 1.5501, + "theoretical_loss": 4.2632986958736625, + "tokens_seen": 237764608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864593781344036, + "loss": 1.788, + "theoretical_loss": 4.263012560832733, + "tokens_seen": 237895680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686258776328987, + "loss": 1.2767, + "theoretical_loss": 4.262726627512886, + "tokens_seen": 238026752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686058174523571, + "loss": 1.7164, + "theoretical_loss": 4.26244089566095, + "tokens_seen": 238157824 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046858575727181545, + "loss": 1.6782, + "theoretical_loss": 4.262155365024207, + "tokens_seen": 238288896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656970912738, + "loss": 1.6008, + "theoretical_loss": 4.261870035350399, + "tokens_seen": 238419968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685456369107322, + "loss": 1.5916, + "theoretical_loss": 4.26158490638772, + "tokens_seen": 238551040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685255767301906, + "loss": 1.6809, + "theoretical_loss": 4.261299977884816, + "tokens_seen": 238682112 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046850551654964895, + "loss": 1.5857, + "theoretical_loss": 4.261015249590789, + "tokens_seen": 238813184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684854563691073, + "loss": 1.5282, + "theoretical_loss": 4.260730721255191, + "tokens_seen": 238944256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046846539618856573, + "loss": 1.7735, + "theoretical_loss": 4.260446392628026, + "tokens_seen": 239075328 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.43187928199768066, + "objective/train/avg_token_score": 0.025535602122545242, + "objective/train/docs_used": 147039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7066162824630737, + "objective/train/original_loss": 4.2467427253723145, + "objective/train/theoretical_loss": 4.260162263459744, + "objective/train/tokens_used": 239206400, + "objective/train/unlikelihood_loss": 0.11330122500658035, + "theoretical_loss": 4.260162263459744, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684453360080241, + "loss": 1.475, + "theoretical_loss": 4.260162263459744, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046842527582748245, + "loss": 1.5486, + "theoretical_loss": 4.259878333501247, + "tokens_seen": 239337472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684052156469408, + "loss": 1.4455, + "theoretical_loss": 4.259594602503881, + "tokens_seen": 239468544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838515546639923, + "loss": 1.6137, + "theoretical_loss": 4.259311070219441, + "tokens_seen": 239599616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683650952858576, + "loss": 1.4483, + "theoretical_loss": 4.259027736400165, + "tokens_seen": 239730688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834503510531595, + "loss": 1.618, + "theoretical_loss": 4.258744600798739, + "tokens_seen": 239861760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683249749247743, + "loss": 1.6814, + "theoretical_loss": 4.258461663168285, + "tokens_seen": 239992832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683049147442327, + "loss": 1.5598, + "theoretical_loss": 4.258178923262376, + "tokens_seen": 240123904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682848545636911, + "loss": 1.6794, + "theoretical_loss": 4.257896380835018, + "tokens_seen": 240254976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046826479438314946, + "loss": 1.5066, + "theoretical_loss": 4.257614035640662, + "tokens_seen": 240386048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682447342026078, + "loss": 1.7484, + "theoretical_loss": 4.257331887434198, + "tokens_seen": 240517120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682246740220662, + "loss": 1.6482, + "theoretical_loss": 4.25704993597095, + "tokens_seen": 240648192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682046138415246, + "loss": 1.5145, + "theoretical_loss": 4.256768181006683, + "tokens_seen": 240779264 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.35654935240745544, + "objective/train/avg_token_score": 0.024610688909888268, + "objective/train/docs_used": 148118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2514936923980713, + "objective/train/original_loss": 3.9919378757476807, + "objective/train/theoretical_loss": 4.2566273771354615, + "objective/train/tokens_used": 240844800, + "objective/train/unlikelihood_loss": 0.20598264038562775, + "theoretical_loss": 4.2566273771354615, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046818455366098296, + "loss": 1.5226, + "theoretical_loss": 4.256486622297595, + "tokens_seen": 240910336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681644934804413, + "loss": 1.7672, + "theoretical_loss": 4.256205259600321, + "tokens_seen": 241041408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681444332998997, + "loss": 1.6025, + "theoretical_loss": 4.255924092671931, + "tokens_seen": 241172480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046812437311935805, + "loss": 1.5735, + "theoretical_loss": 4.255643121269924, + "tokens_seen": 241303552 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810431293881646, + "loss": 1.8846, + "theoretical_loss": 4.255362345152234, + "tokens_seen": 241434624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680842527582748, + "loss": 1.5332, + "theoretical_loss": 4.255081764077224, + "tokens_seen": 241565696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680641925777332, + "loss": 1.6185, + "theoretical_loss": 4.254801377803689, + "tokens_seen": 241696768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046804413239719155, + "loss": 1.5774, + "theoretical_loss": 4.254521186090852, + "tokens_seen": 241827840 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802407221664997, + "loss": 1.728, + "theoretical_loss": 4.254241188698361, + "tokens_seen": 241958912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800401203610833, + "loss": 1.5526, + "theoretical_loss": 4.2539613853862965, + "tokens_seen": 242089984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679839518555667, + "loss": 1.6072, + "theoretical_loss": 4.253681775915161, + "tokens_seen": 242221056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046796389167502505, + "loss": 1.7814, + "theoretical_loss": 4.253402360045882, + "tokens_seen": 242352128 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.2602028250694275, + "objective/train/avg_token_score": 0.019304223358631134, + "objective/train/docs_used": 148826, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9984758496284485, + "objective/train/original_loss": 4.527791500091553, + "objective/train/theoretical_loss": 4.253123137539814, + "objective/train/tokens_used": 242483200, + "objective/train/unlikelihood_loss": 0.15593910217285156, + "theoretical_loss": 4.253123137539814, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679438314944834, + "loss": 1.4314, + "theoretical_loss": 4.253123137539814, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046792377131394183, + "loss": 1.5239, + "theoretical_loss": 4.25284410815873, + "tokens_seen": 242614272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679037111334002, + "loss": 1.5362, + "theoretical_loss": 4.252565271664828, + "tokens_seen": 242745344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046788365095285856, + "loss": 1.663, + "theoretical_loss": 4.252286627820727, + "tokens_seen": 242876416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678635907723169, + "loss": 1.4544, + "theoretical_loss": 4.252008176389465, + "tokens_seen": 243007488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046784353059177534, + "loss": 1.5947, + "theoretical_loss": 4.251729917134498, + "tokens_seen": 243138560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678234704112337, + "loss": 1.4033, + "theoretical_loss": 4.251451849819704, + "tokens_seen": 243269632 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046780341023069206, + "loss": 1.618, + "theoretical_loss": 4.251173974209375, + "tokens_seen": 243400704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677833500501504, + "loss": 1.5327, + "theoretical_loss": 4.250896290068218, + "tokens_seen": 243531776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677632898696088, + "loss": 1.7856, + "theoretical_loss": 4.25061879716136, + "tokens_seen": 243662848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677432296890672, + "loss": 1.9125, + "theoretical_loss": 4.250341495254337, + "tokens_seen": 243793920 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046772316950852556, + "loss": 1.6721, + "theoretical_loss": 4.250064384113102, + "tokens_seen": 243924992 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467703109327984, + "loss": 1.4838, + "theoretical_loss": 4.249787463504019, + "tokens_seen": 244056064 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.20821113884449005, + "objective/train/avg_token_score": 0.05237054452300072, + "objective/train/docs_used": 150000, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8499055504798889, + "objective/train/original_loss": 5.0128703117370605, + "objective/train/theoretical_loss": 4.249649074576134, + "objective/train/tokens_used": 244121600, + "objective/train/unlikelihood_loss": 0.0956830307841301, + "theoretical_loss": 4.249649074576134, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046768304914744234, + "loss": 1.5865, + "theoretical_loss": 4.249510733193862, + "tokens_seen": 244187136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046766298896690076, + "loss": 1.6801, + "theoretical_loss": 4.249234192949818, + "tokens_seen": 244318208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676429287863591, + "loss": 1.5263, + "theoretical_loss": 4.248957842539484, + "tokens_seen": 244449280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676228686058175, + "loss": 1.5385, + "theoretical_loss": 4.2486816817308615, + "tokens_seen": 244580352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046760280842527585, + "loss": 1.5574, + "theoretical_loss": 4.248405710292364, + "tokens_seen": 244711424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675827482447342, + "loss": 1.5474, + "theoretical_loss": 4.248129927992808, + "tokens_seen": 244842496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675626880641926, + "loss": 1.64, + "theoretical_loss": 4.247854334601421, + "tokens_seen": 244973568 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467542627883651, + "loss": 1.6484, + "theoretical_loss": 4.247578929887829, + "tokens_seen": 245104640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046752256770310935, + "loss": 1.5901, + "theoretical_loss": 4.247303713622067, + "tokens_seen": 245235712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675025075225677, + "loss": 1.697, + "theoretical_loss": 4.247028685574569, + "tokens_seen": 245366784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046748244734202613, + "loss": 1.8329, + "theoretical_loss": 4.246753845516174, + "tokens_seen": 245497856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674623871614845, + "loss": 1.582, + "theoretical_loss": 4.246479193218123, + "tokens_seen": 245628928 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.35355570912361145, + "objective/train/avg_token_score": 0.023441534489393234, + "objective/train/docs_used": 150657, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.178994059562683, + "objective/train/original_loss": 3.2855653762817383, + "objective/train/theoretical_loss": 4.246204728452055, + "objective/train/tokens_used": 245760000, + "objective/train/unlikelihood_loss": 0.2683996260166168, + "theoretical_loss": 4.246204728452055, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046744232698094285, + "loss": 1.4603, + "theoretical_loss": 4.246204728452055, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674222668004012, + "loss": 1.5375, + "theoretical_loss": 4.245930450990007, + "tokens_seen": 245891072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046740220661985963, + "loss": 1.4696, + "theoretical_loss": 4.245656360604417, + "tokens_seen": 246022144 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467382146439318, + "loss": 1.641, + "theoretical_loss": 4.2453824570681205, + "tokens_seen": 246153216 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046736208625877636, + "loss": 1.5838, + "theoretical_loss": 4.2451087401543495, + "tokens_seen": 246284288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673420260782347, + "loss": 1.736, + "theoretical_loss": 4.24483520963673, + "tokens_seen": 246415360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673219658976931, + "loss": 1.4829, + "theoretical_loss": 4.244561865289285, + "tokens_seen": 246546432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673019057171515, + "loss": 1.6817, + "theoretical_loss": 4.2442887068864295, + "tokens_seen": 246677504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046728184553660986, + "loss": 1.5743, + "theoretical_loss": 4.244015734202973, + "tokens_seen": 246808576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672617853560682, + "loss": 1.6287, + "theoretical_loss": 4.243742947014117, + "tokens_seen": 246939648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672417251755266, + "loss": 1.5413, + "theoretical_loss": 4.243470345095453, + "tokens_seen": 247070720 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467221664994985, + "loss": 1.5116, + "theoretical_loss": 4.243197928222964, + "tokens_seen": 247201792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046720160481444336, + "loss": 1.6491, + "theoretical_loss": 4.242925696173021, + "tokens_seen": 247332864 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.2963709533214569, + "objective/train/avg_token_score": 0.031853631138801575, + "objective/train/docs_used": 151862, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1483128070831299, + "objective/train/original_loss": 4.083296775817871, + "objective/train/theoretical_loss": 4.2427896493867285, + "objective/train/tokens_used": 247398400, + "objective/train/unlikelihood_loss": 0.13969635963439941, + "theoretical_loss": 4.2427896493867285, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671815446339017, + "loss": 1.597, + "theoretical_loss": 4.242653648722387, + "tokens_seen": 247463936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671614844533601, + "loss": 1.5181, + "theoretical_loss": 4.24238178564821, + "tokens_seen": 247595008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046714142427281845, + "loss": 1.7096, + "theoretical_loss": 4.242110106728022, + "tokens_seen": 247726080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046712136409227686, + "loss": 1.7045, + "theoretical_loss": 4.241838611739748, + "tokens_seen": 247857152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671013039117352, + "loss": 1.4473, + "theoretical_loss": 4.241567300461693, + "tokens_seen": 247988224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670812437311936, + "loss": 1.4892, + "theoretical_loss": 4.241296172672547, + "tokens_seen": 248119296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046706118355065195, + "loss": 1.4327, + "theoretical_loss": 4.241025228151383, + "tokens_seen": 248250368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046704112337011037, + "loss": 1.4395, + "theoretical_loss": 4.240754466677659, + "tokens_seen": 248381440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046702106318956873, + "loss": 1.3767, + "theoretical_loss": 4.240483888031212, + "tokens_seen": 248512512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670010030090271, + "loss": 1.6805, + "theoretical_loss": 4.240213491992261, + "tokens_seen": 248643584 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046698094282848545, + "loss": 1.7161, + "theoretical_loss": 4.239943278341404, + "tokens_seen": 248774656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669608826479438, + "loss": 1.6768, + "theoretical_loss": 4.239673246859619, + "tokens_seen": 248905728 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.40518084168434143, + "objective/train/avg_token_score": 0.00906757079064846, + "objective/train/docs_used": 152583, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6137572526931763, + "objective/train/original_loss": 3.96043062210083, + "objective/train/theoretical_loss": 4.239403397328261, + "objective/train/tokens_used": 249036800, + "objective/train/unlikelihood_loss": 0.250114768743515, + "theoretical_loss": 4.239403397328261, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694082246740223, + "loss": 1.6961, + "theoretical_loss": 4.239403397328261, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669207622868606, + "loss": 1.5261, + "theoretical_loss": 4.239133729529064, + "tokens_seen": 249167872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046690070210631896, + "loss": 1.7208, + "theoretical_loss": 4.2388642432441355, + "tokens_seen": 249298944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668806419257773, + "loss": 1.4695, + "theoretical_loss": 4.238594938255963, + "tokens_seen": 249430016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046686058174523574, + "loss": 1.5757, + "theoretical_loss": 4.238325814347404, + "tokens_seen": 249561088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668405215646941, + "loss": 1.5495, + "theoretical_loss": 4.238056871301695, + "tokens_seen": 249692160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682046138415246, + "loss": 1.4589, + "theoretical_loss": 4.237788108902441, + "tokens_seen": 249823232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668004012036108, + "loss": 1.5079, + "theoretical_loss": 4.237519526933622, + "tokens_seen": 249954304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667803410230692, + "loss": 1.6051, + "theoretical_loss": 4.237251125179588, + "tokens_seen": 250085376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667602808425276, + "loss": 1.6881, + "theoretical_loss": 4.23698290342506, + "tokens_seen": 250216448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046674022066198596, + "loss": 1.4733, + "theoretical_loss": 4.23671486145513, + "tokens_seen": 250347520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667201604814443, + "loss": 1.4943, + "theoretical_loss": 4.236446999055257, + "tokens_seen": 250478592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667001003009027, + "loss": 1.5532, + "theoretical_loss": 4.2361793160112695, + "tokens_seen": 250609664 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.38782989978790283, + "objective/train/avg_token_score": 0.009073655121028423, + "objective/train/docs_used": 153242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.569201111793518, + "objective/train/original_loss": 4.112845420837402, + "objective/train/theoretical_loss": 4.236045541680905, + "objective/train/tokens_used": 250675200, + "objective/train/unlikelihood_loss": 0.14907237887382507, + "theoretical_loss": 4.236045541680905, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666800401203611, + "loss": 1.5928, + "theoretical_loss": 4.235911812109363, + "tokens_seen": 250740736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046665997993981947, + "loss": 1.5379, + "theoretical_loss": 4.235644487136098, + "tokens_seen": 250871808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046663991975927783, + "loss": 1.5322, + "theoretical_loss": 4.235377340878404, + "tokens_seen": 251002880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666198595787362, + "loss": 1.4665, + "theoretical_loss": 4.235110373123572, + "tokens_seen": 251133952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046659979939819455, + "loss": 1.4523, + "theoretical_loss": 4.2348435836592575, + "tokens_seen": 251265024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046657973921765297, + "loss": 1.6582, + "theoretical_loss": 4.234576972273481, + "tokens_seen": 251396096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046655967903711133, + "loss": 1.662, + "theoretical_loss": 4.234310538754624, + "tokens_seen": 251527168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665396188565697, + "loss": 1.4291, + "theoretical_loss": 4.234044282891429, + "tokens_seen": 251658240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046651955867602806, + "loss": 1.4925, + "theoretical_loss": 4.233778204473002, + "tokens_seen": 251789312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664994984954865, + "loss": 1.6234, + "theoretical_loss": 4.233512303288807, + "tokens_seen": 251920384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046647943831494484, + "loss": 1.5104, + "theoretical_loss": 4.233246579128666, + "tokens_seen": 252051456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664593781344032, + "loss": 1.5029, + "theoretical_loss": 4.232981031782761, + "tokens_seen": 252182528 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.3658357858657837, + "objective/train/avg_token_score": 0.005103480536490679, + "objective/train/docs_used": 154565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2179532051086426, + "objective/train/original_loss": 3.7866077423095703, + "objective/train/theoretical_loss": 4.232715661041632, + "objective/train/tokens_used": 252313600, + "objective/train/unlikelihood_loss": 0.20572546124458313, + "theoretical_loss": 4.232715661041632, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046643931795386156, + "loss": 1.6638, + "theoretical_loss": 4.232715661041632, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664192577733199, + "loss": 1.4777, + "theoretical_loss": 4.232450466696174, + "tokens_seen": 252444672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046639919759277834, + "loss": 1.4621, + "theoretical_loss": 4.23218544853764, + "tokens_seen": 252575744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663791374122367, + "loss": 1.815, + "theoretical_loss": 4.231920606357638, + "tokens_seen": 252706816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046635907723169506, + "loss": 1.5611, + "theoretical_loss": 4.231655939948127, + "tokens_seen": 252837888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663390170511534, + "loss": 1.6306, + "theoretical_loss": 4.231391449101425, + "tokens_seen": 252968960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046631895687061184, + "loss": 1.5308, + "theoretical_loss": 4.231127133610198, + "tokens_seen": 253100032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662988966900702, + "loss": 1.4716, + "theoretical_loss": 4.230862993267468, + "tokens_seen": 253231104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046627883650952857, + "loss": 1.5696, + "theoretical_loss": 4.230599027866606, + "tokens_seen": 253362176 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046625877632898693, + "loss": 1.6201, + "theoretical_loss": 4.2303352372013325, + "tokens_seen": 253493248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046623871614844535, + "loss": 1.5034, + "theoretical_loss": 4.230071621065721, + "tokens_seen": 253624320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662186559679037, + "loss": 1.422, + "theoretical_loss": 4.229808179254192, + "tokens_seen": 253755392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661985957873621, + "loss": 1.49, + "theoretical_loss": 4.229544911561513, + "tokens_seen": 253886464 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.5327468514442444, + "objective/train/avg_token_score": 0.0074698301032185555, + "objective/train/docs_used": 156106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.954049825668335, + "objective/train/original_loss": 3.7289023399353027, + "objective/train/theoretical_loss": 4.229413342945703, + "objective/train/tokens_used": 253952000, + "objective/train/unlikelihood_loss": 0.149134561419487, + "theoretical_loss": 4.229413342945703, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661785356068205, + "loss": 1.7898, + "theoretical_loss": 4.229281817782801, + "tokens_seen": 254017536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046615847542627885, + "loss": 1.6864, + "theoretical_loss": 4.229018897713519, + "tokens_seen": 254148608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046613841524573727, + "loss": 1.3615, + "theoretical_loss": 4.228756151149475, + "tokens_seen": 254279680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046611835506519563, + "loss": 1.615, + "theoretical_loss": 4.228493577886824, + "tokens_seen": 254410752 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466098294884654, + "loss": 1.5096, + "theoretical_loss": 4.228231177722063, + "tokens_seen": 254541824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046607823470411235, + "loss": 1.4544, + "theoretical_loss": 4.227968950452035, + "tokens_seen": 254672896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046605817452357077, + "loss": 1.661, + "theoretical_loss": 4.227706895873924, + "tokens_seen": 254803968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046603811434302913, + "loss": 1.4282, + "theoretical_loss": 4.227445013785257, + "tokens_seen": 254935040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660180541624875, + "loss": 1.6551, + "theoretical_loss": 4.227183303983901, + "tokens_seen": 255066112 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046599799398194586, + "loss": 1.4951, + "theoretical_loss": 4.226921766268067, + "tokens_seen": 255197184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659779338014042, + "loss": 1.449, + "theoretical_loss": 4.226660400436302, + "tokens_seen": 255328256 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595787362086263, + "loss": 1.5032, + "theoretical_loss": 4.226399206287493, + "tokens_seen": 255459328 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4268695116043091, + "objective/train/avg_token_score": 0.012001312337815762, + "objective/train/docs_used": 156630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8062608242034912, + "objective/train/original_loss": 4.282796382904053, + "objective/train/theoretical_loss": 4.226138183620867, + "objective/train/tokens_used": 255590400, + "objective/train/unlikelihood_loss": 0.11538192629814148, + "theoretical_loss": 4.226138183620867, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465937813440321, + "loss": 1.5151, + "theoretical_loss": 4.226138183620867, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591775325977936, + "loss": 1.6603, + "theoretical_loss": 4.225877332235987, + "tokens_seen": 255721472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658976930792377, + "loss": 1.4763, + "theoretical_loss": 4.225616651932753, + "tokens_seen": 255852544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046587763289869614, + "loss": 1.6467, + "theoretical_loss": 4.225356142511402, + "tokens_seen": 255983616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658575727181545, + "loss": 1.5337, + "theoretical_loss": 4.225095803772507, + "tokens_seen": 256114688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046583751253761286, + "loss": 1.5934, + "theoretical_loss": 4.224835635516973, + "tokens_seen": 256245760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658174523570712, + "loss": 1.4188, + "theoretical_loss": 4.224575637546041, + "tokens_seen": 256376832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657973921765296, + "loss": 1.3615, + "theoretical_loss": 4.2243158096612845, + "tokens_seen": 256507904 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465777331995988, + "loss": 1.7877, + "theoretical_loss": 4.22405615166461, + "tokens_seen": 256638976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575727181544636, + "loss": 1.652, + "theoretical_loss": 4.223796663358255, + "tokens_seen": 256770048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657372116349047, + "loss": 1.5669, + "theoretical_loss": 4.223537344544788, + "tokens_seen": 256901120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171514543631, + "loss": 1.5028, + "theoretical_loss": 4.22327819502711, + "tokens_seen": 257032192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656970912738215, + "loss": 1.5612, + "theoretical_loss": 4.223019214608446, + "tokens_seen": 257163264 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.35031768679618835, + "objective/train/avg_token_score": 0.06552369147539139, + "objective/train/docs_used": 157258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4199466705322266, + "objective/train/original_loss": 4.115721225738525, + "objective/train/theoretical_loss": 4.222889787749845, + "objective/train/tokens_used": 257228800, + "objective/train/unlikelihood_loss": 0.19470366835594177, + "theoretical_loss": 4.222889787749845, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046567703109327987, + "loss": 1.5506, + "theoretical_loss": 4.222760403092358, + "tokens_seen": 257294336 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565697091273823, + "loss": 1.5487, + "theoretical_loss": 4.222501760282729, + "tokens_seen": 257425408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656369107321966, + "loss": 1.5823, + "theoretical_loss": 4.222243285983772, + "tokens_seen": 257556480 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046561685055165495, + "loss": 1.6221, + "theoretical_loss": 4.221984980000029, + "tokens_seen": 257687552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046559679037111337, + "loss": 1.7549, + "theoretical_loss": 4.221726842136364, + "tokens_seen": 257818624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046557673019057173, + "loss": 1.7467, + "theoretical_loss": 4.221468872197967, + "tokens_seen": 257949696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655566700100301, + "loss": 1.6515, + "theoretical_loss": 4.221211069990357, + "tokens_seen": 258080768 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553660982948846, + "loss": 1.5346, + "theoretical_loss": 4.22095343531937, + "tokens_seen": 258211840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655165496489469, + "loss": 1.6984, + "theoretical_loss": 4.220695967991171, + "tokens_seen": 258342912 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046549648946840524, + "loss": 1.5019, + "theoretical_loss": 4.220438667812244, + "tokens_seen": 258473984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654764292878636, + "loss": 1.5388, + "theoretical_loss": 4.220181534589398, + "tokens_seen": 258605056 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046545636910732196, + "loss": 1.7965, + "theoretical_loss": 4.219924568129759, + "tokens_seen": 258736128 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4152003824710846, + "objective/train/avg_token_score": 0.016226625069975853, + "objective/train/docs_used": 158510, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.525114893913269, + "objective/train/original_loss": 4.261040210723877, + "objective/train/theoretical_loss": 4.219667768240775, + "objective/train/tokens_used": 258867200, + "objective/train/unlikelihood_loss": 0.1502227634191513, + "theoretical_loss": 4.219667768240775, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654363089267803, + "loss": 1.6609, + "theoretical_loss": 4.219667768240775, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046541624874623874, + "loss": 1.5927, + "theoretical_loss": 4.2194111347302155, + "tokens_seen": 258998272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653961885656971, + "loss": 1.5297, + "theoretical_loss": 4.219154667406166, + "tokens_seen": 259129344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046537612838515546, + "loss": 1.5517, + "theoretical_loss": 4.218898366077035, + "tokens_seen": 259260416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653560682046138, + "loss": 1.6273, + "theoretical_loss": 4.218642230551541, + "tokens_seen": 259391488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533600802407224, + "loss": 1.478, + "theoretical_loss": 4.218386260638727, + "tokens_seen": 259522560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653159478435306, + "loss": 1.6475, + "theoretical_loss": 4.218130456147948, + "tokens_seen": 259653632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046529588766298897, + "loss": 1.3824, + "theoretical_loss": 4.217874816888877, + "tokens_seen": 259784704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046527582748244733, + "loss": 1.474, + "theoretical_loss": 4.217619342671498, + "tokens_seen": 259915776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525576730190575, + "loss": 1.5144, + "theoretical_loss": 4.217364033306113, + "tokens_seen": 260046848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652357071213641, + "loss": 1.7336, + "theoretical_loss": 4.217108888603337, + "tokens_seen": 260177920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046521564694082247, + "loss": 1.577, + "theoretical_loss": 4.216853908374097, + "tokens_seen": 260308992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519558676028083, + "loss": 1.5176, + "theoretical_loss": 4.216599092429631, + "tokens_seen": 260440064 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.37017351388931274, + "objective/train/avg_token_score": 0.00918534118682146, + "objective/train/docs_used": 159098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4167944192886353, + "objective/train/original_loss": 3.8629274368286133, + "objective/train/theoretical_loss": 4.216471746005286, + "objective/train/tokens_used": 260505600, + "objective/train/unlikelihood_loss": 0.2083285003900528, + "theoretical_loss": 4.216471746005286, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651755265797392, + "loss": 1.5863, + "theoretical_loss": 4.216344440581491, + "tokens_seen": 260571136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651554663991976, + "loss": 1.4607, + "theoretical_loss": 4.2160899526415365, + "tokens_seen": 260702208 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465135406218656, + "loss": 1.394, + "theoretical_loss": 4.215835628421942, + "tokens_seen": 260833280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046511534603811434, + "loss": 1.5575, + "theoretical_loss": 4.215581467735187, + "tokens_seen": 260964352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650952858575727, + "loss": 1.5175, + "theoretical_loss": 4.215327470394062, + "tokens_seen": 261095424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650752256770311, + "loss": 1.5183, + "theoretical_loss": 4.215073636211664, + "tokens_seen": 261226496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650551654964895, + "loss": 1.5299, + "theoretical_loss": 4.214819965001401, + "tokens_seen": 261357568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046503510531594784, + "loss": 1.5135, + "theoretical_loss": 4.214566456576984, + "tokens_seen": 261488640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650150451354062, + "loss": 1.4946, + "theoretical_loss": 4.214313110752431, + "tokens_seen": 261619712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046499498495486456, + "loss": 1.4256, + "theoretical_loss": 4.214059927342068, + "tokens_seen": 261750784 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464974924774323, + "loss": 1.473, + "theoretical_loss": 4.213806906160523, + "tokens_seen": 261881856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046495486459378134, + "loss": 1.606, + "theoretical_loss": 4.213554047022729, + "tokens_seen": 262012928 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.30541300773620605, + "objective/train/avg_token_score": 0.026100359857082367, + "objective/train/docs_used": 160106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1186697483062744, + "objective/train/original_loss": 4.202697277069092, + "objective/train/theoretical_loss": 4.213301349743924, + "objective/train/tokens_used": 262144000, + "objective/train/unlikelihood_loss": 0.18503674864768982, + "theoretical_loss": 4.213301349743924, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649348044132397, + "loss": 1.6211, + "theoretical_loss": 4.213301349743924, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046491474423269807, + "loss": 1.5147, + "theoretical_loss": 4.213048814139647, + "tokens_seen": 262275072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648946840521565, + "loss": 1.4764, + "theoretical_loss": 4.21279644002574, + "tokens_seen": 262406144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046487462387161485, + "loss": 1.5803, + "theoretical_loss": 4.212544227218347, + "tokens_seen": 262537216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648545636910732, + "loss": 1.4761, + "theoretical_loss": 4.212292175533912, + "tokens_seen": 262668288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046483450351053157, + "loss": 1.6497, + "theoretical_loss": 4.212040284789181, + "tokens_seen": 262799360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046481444332998993, + "loss": 1.4949, + "theoretical_loss": 4.211788554801198, + "tokens_seen": 262930432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046479438314944835, + "loss": 1.6391, + "theoretical_loss": 4.211536985387307, + "tokens_seen": 263061504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647743229689067, + "loss": 1.4038, + "theoretical_loss": 4.2112855763651496, + "tokens_seen": 263192576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647542627883651, + "loss": 1.5316, + "theoretical_loss": 4.211034327552666, + "tokens_seen": 263323648 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046473420260782344, + "loss": 1.4309, + "theoretical_loss": 4.210783238768093, + "tokens_seen": 263454720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647141424272819, + "loss": 1.5506, + "theoretical_loss": 4.210532309829965, + "tokens_seen": 263585792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046469408224674027, + "loss": 1.5781, + "theoretical_loss": 4.21028154055711, + "tokens_seen": 263716864 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.32484114170074463, + "objective/train/avg_token_score": 0.0183444544672966, + "objective/train/docs_used": 160649, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2061073780059814, + "objective/train/original_loss": 4.3018927574157715, + "objective/train/theoretical_loss": 4.2101562157386265, + "objective/train/tokens_used": 263782400, + "objective/train/unlikelihood_loss": 0.16418404877185822, + "theoretical_loss": 4.2101562157386265, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046467402206619863, + "loss": 1.4389, + "theoretical_loss": 4.210030930768655, + "tokens_seen": 263847936 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464653961885657, + "loss": 1.445, + "theoretical_loss": 4.209780480284017, + "tokens_seen": 263979008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046463390170511535, + "loss": 1.5965, + "theoretical_loss": 4.209530188922911, + "tokens_seen": 264110080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046461384152457377, + "loss": 1.4428, + "theoretical_loss": 4.209280056505342, + "tokens_seen": 264241152 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046459378134403213, + "loss": 1.3129, + "theoretical_loss": 4.209030082851612, + "tokens_seen": 264372224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645737211634905, + "loss": 1.498, + "theoretical_loss": 4.20878026778231, + "tokens_seen": 264503296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046455366098294886, + "loss": 1.5519, + "theoretical_loss": 4.208530611118321, + "tokens_seen": 264634368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645336008024073, + "loss": 1.5022, + "theoretical_loss": 4.208281112680817, + "tokens_seen": 264765440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046451354062186564, + "loss": 1.5408, + "theoretical_loss": 4.208031772291265, + "tokens_seen": 264896512 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464493480441324, + "loss": 1.2914, + "theoretical_loss": 4.2077825897714165, + "tokens_seen": 265027584 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046447342026078236, + "loss": 1.4717, + "theoretical_loss": 4.207533564943316, + "tokens_seen": 265158656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644533600802407, + "loss": 1.5337, + "theoretical_loss": 4.2072846976292935, + "tokens_seen": 265289728 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.37787145376205444, + "objective/train/avg_token_score": 0.033646829426288605, + "objective/train/docs_used": 161970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4991329908370972, + "objective/train/original_loss": 4.190028667449951, + "objective/train/theoretical_loss": 4.20703598765197, + "objective/train/tokens_used": 265420800, + "objective/train/unlikelihood_loss": 0.1276427060365677, + "theoretical_loss": 4.20703598765197, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046443329989969914, + "loss": 1.5316, + "theoretical_loss": 4.20703598765197, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644132397191575, + "loss": 1.5589, + "theoretical_loss": 4.206787434834251, + "tokens_seen": 265551872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046439317953861586, + "loss": 1.4447, + "theoretical_loss": 4.206539038999329, + "tokens_seen": 265682944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643731193580742, + "loss": 1.411, + "theoretical_loss": 4.206290799970685, + "tokens_seen": 265814016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046435305917753264, + "loss": 1.4428, + "theoretical_loss": 4.206042717572082, + "tokens_seen": 265945088 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464332998996991, + "loss": 1.4921, + "theoretical_loss": 4.20579479162757, + "tokens_seen": 266076160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046431293881644937, + "loss": 1.487, + "theoretical_loss": 4.205547021961482, + "tokens_seen": 266207232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046429287863590773, + "loss": 1.5737, + "theoretical_loss": 4.205299408398435, + "tokens_seen": 266338304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046427281845536615, + "loss": 1.6461, + "theoretical_loss": 4.20505195076333, + "tokens_seen": 266469376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642527582748245, + "loss": 1.7422, + "theoretical_loss": 4.20480464888135, + "tokens_seen": 266600448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046423269809428287, + "loss": 1.4776, + "theoretical_loss": 4.204557502577957, + "tokens_seen": 266731520 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046421263791374123, + "loss": 1.63, + "theoretical_loss": 4.2043105116789, + "tokens_seen": 266862592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641925777331996, + "loss": 1.7364, + "theoretical_loss": 4.204063676010202, + "tokens_seen": 266993664 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.494012713432312, + "objective/train/avg_token_score": 0.00496962945908308, + "objective/train/docs_used": 162606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7318894863128662, + "objective/train/original_loss": 3.5257256031036377, + "objective/train/theoretical_loss": 4.203940316332948, + "objective/train/tokens_used": 267059200, + "objective/train/unlikelihood_loss": 0.2108738273382187, + "theoretical_loss": 4.203940316332948, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464172517552658, + "loss": 1.3925, + "theoretical_loss": 4.203816995398171, + "tokens_seen": 267124736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641524573721164, + "loss": 1.6141, + "theoretical_loss": 4.203570469669392, + "tokens_seen": 267255808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046413239719157474, + "loss": 1.608, + "theoretical_loss": 4.203324098650731, + "tokens_seen": 267386880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641123370110331, + "loss": 1.761, + "theoretical_loss": 4.20307788216933, + "tokens_seen": 267517952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640922768304915, + "loss": 1.636, + "theoretical_loss": 4.202831820052609, + "tokens_seen": 267649024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640722166499499, + "loss": 1.5856, + "theoretical_loss": 4.202585912128266, + "tokens_seen": 267780096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046405215646940824, + "loss": 1.6837, + "theoretical_loss": 4.202340158224277, + "tokens_seen": 267911168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640320962888666, + "loss": 1.6194, + "theoretical_loss": 4.2020945581688895, + "tokens_seen": 268042240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046401203610832496, + "loss": 1.7856, + "theoretical_loss": 4.201849111790631, + "tokens_seen": 268173312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639919759277834, + "loss": 1.6562, + "theoretical_loss": 4.201603818918302, + "tokens_seen": 268304384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046397191574724174, + "loss": 1.5358, + "theoretical_loss": 4.201358679380976, + "tokens_seen": 268435456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639518555667001, + "loss": 1.6419, + "theoretical_loss": 4.201113693008002, + "tokens_seen": 268566528 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.3462243378162384, + "objective/train/avg_token_score": 0.018634682521224022, + "objective/train/docs_used": 163909, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2885785102844238, + "objective/train/original_loss": 4.379140853881836, + "objective/train/theoretical_loss": 4.2008688596290025, + "objective/train/tokens_used": 268697600, + "objective/train/unlikelihood_loss": 0.12308195978403091, + "theoretical_loss": 4.2008688596290025, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393179538615847, + "loss": 1.5429, + "theoretical_loss": 4.2008688596290025, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639117352056169, + "loss": 1.596, + "theoretical_loss": 4.200624179073869, + "tokens_seen": 268828672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389167502507525, + "loss": 1.6086, + "theoretical_loss": 4.200379651172769, + "tokens_seen": 268959744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638716148445336, + "loss": 1.4534, + "theoretical_loss": 4.200135275756139, + "tokens_seen": 269090816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385155466399197, + "loss": 1.6596, + "theoretical_loss": 4.199891052654689, + "tokens_seen": 269221888 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046383149448345033, + "loss": 1.4265, + "theoretical_loss": 4.199646981699395, + "tokens_seen": 269352960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381143430290875, + "loss": 1.3837, + "theoretical_loss": 4.199403062721506, + "tokens_seen": 269484032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637913741223671, + "loss": 1.5549, + "theoretical_loss": 4.1991592955525405, + "tokens_seen": 269615104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637713139418255, + "loss": 1.5172, + "theoretical_loss": 4.198915680024282, + "tokens_seen": 269746176 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375125376128384, + "loss": 1.5907, + "theoretical_loss": 4.198672215968785, + "tokens_seen": 269877248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046373119358074225, + "loss": 1.5905, + "theoretical_loss": 4.198428903218371, + "tokens_seen": 270008320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637111334002006, + "loss": 1.5348, + "theoretical_loss": 4.198185741605628, + "tokens_seen": 270139392 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463691073219659, + "loss": 1.4577, + "theoretical_loss": 4.197942730963412, + "tokens_seen": 270270464 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.5173509120941162, + "objective/train/avg_token_score": 0.017678294330835342, + "objective/train/docs_used": 164570, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0276877880096436, + "objective/train/original_loss": 4.127265930175781, + "objective/train/theoretical_loss": 4.19782128220409, + "objective/train/tokens_used": 270336000, + "objective/train/unlikelihood_loss": 0.1530839055776596, + "theoretical_loss": 4.19782128220409, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046367101303911734, + "loss": 1.426, + "theoretical_loss": 4.19769987112484, + "tokens_seen": 270401536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636509528585757, + "loss": 1.4833, + "theoretical_loss": 4.1974571619233, + "tokens_seen": 270532608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636308926780341, + "loss": 1.5496, + "theoretical_loss": 4.1972146031924416, + "tokens_seen": 270663680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636108324974925, + "loss": 1.6277, + "theoretical_loss": 4.196972194766179, + "tokens_seen": 270794752 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359077231695084, + "loss": 1.5659, + "theoretical_loss": 4.1967299364786905, + "tokens_seen": 270925824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635707121364092, + "loss": 1.6559, + "theoretical_loss": 4.1964878281644165, + "tokens_seen": 271056896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635506519558676, + "loss": 1.5899, + "theoretical_loss": 4.196245869658061, + "tokens_seen": 271187968 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463530591775326, + "loss": 1.6648, + "theoretical_loss": 4.196004060794589, + "tokens_seen": 271319040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046351053159478434, + "loss": 1.5551, + "theoretical_loss": 4.195762401409229, + "tokens_seen": 271450112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634904714142427, + "loss": 1.6318, + "theoretical_loss": 4.195520891337466, + "tokens_seen": 271581184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046347041123370107, + "loss": 1.7221, + "theoretical_loss": 4.19527953041505, + "tokens_seen": 271712256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634503510531595, + "loss": 1.3852, + "theoretical_loss": 4.195038318477989, + "tokens_seen": 271843328 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.5086143612861633, + "objective/train/avg_token_score": 0.018605908378958702, + "objective/train/docs_used": 165961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.964569330215454, + "objective/train/original_loss": 4.285537242889404, + "objective/train/theoretical_loss": 4.194797255362549, + "objective/train/tokens_used": 271974400, + "objective/train/unlikelihood_loss": 0.10453181713819504, + "theoretical_loss": 4.194797255362549, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343029087261785, + "loss": 1.6417, + "theoretical_loss": 4.194797255362549, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634102306920762, + "loss": 1.5556, + "theoretical_loss": 4.194556340905256, + "tokens_seen": 272105472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046339017051153457, + "loss": 1.6563, + "theoretical_loss": 4.194315574942896, + "tokens_seen": 272236544 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463370110330993, + "loss": 1.6543, + "theoretical_loss": 4.19407495731251, + "tokens_seen": 272367616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046335005015045135, + "loss": 1.608, + "theoretical_loss": 4.193834487851396, + "tokens_seen": 272498688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633299899699097, + "loss": 1.4458, + "theoretical_loss": 4.1935941663971095, + "tokens_seen": 272629760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633099297893681, + "loss": 1.5378, + "theoretical_loss": 4.193353992787463, + "tokens_seen": 272760832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046328986960882644, + "loss": 1.3823, + "theoretical_loss": 4.1931139668605235, + "tokens_seen": 272891904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046326980942828485, + "loss": 1.47, + "theoretical_loss": 4.192874088454613, + "tokens_seen": 273022976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632497492477432, + "loss": 1.508, + "theoretical_loss": 4.192634357408309, + "tokens_seen": 273154048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632296890672016, + "loss": 1.5608, + "theoretical_loss": 4.192394773560441, + "tokens_seen": 273285120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046320962888666, + "loss": 1.5031, + "theoretical_loss": 4.192155336750094, + "tokens_seen": 273416192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631895687061184, + "loss": 1.8175, + "theoretical_loss": 4.191916046816605, + "tokens_seen": 273547264 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.39100682735443115, + "objective/train/avg_token_score": 0.018308745697140694, + "objective/train/docs_used": 166474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.588956356048584, + "objective/train/original_loss": 4.311248779296875, + "objective/train/theoretical_loss": 4.191796456878544, + "objective/train/tokens_used": 273612800, + "objective/train/unlikelihood_loss": 0.15257127583026886, + "theoretical_loss": 4.191796456878544, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631695085255768, + "loss": 1.6023, + "theoretical_loss": 4.191676903599563, + "tokens_seen": 273678336 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046314944834503514, + "loss": 1.4118, + "theoretical_loss": 4.191437906938811, + "tokens_seen": 273809408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631293881644935, + "loss": 1.557, + "theoretical_loss": 4.19119905667444, + "tokens_seen": 273940480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631093279839519, + "loss": 1.3291, + "theoretical_loss": 4.190960352646796, + "tokens_seen": 274071552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630892678034103, + "loss": 1.3456, + "theoretical_loss": 4.19072179469647, + "tokens_seen": 274202624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046306920762286864, + "loss": 1.3644, + "theoretical_loss": 4.190483382664308, + "tokens_seen": 274333696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463049147442327, + "loss": 1.624, + "theoretical_loss": 4.190245116391403, + "tokens_seen": 274464768 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046302908726178536, + "loss": 1.5528, + "theoretical_loss": 4.190006995719098, + "tokens_seen": 274595840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630090270812438, + "loss": 1.4449, + "theoretical_loss": 4.189769020488981, + "tokens_seen": 274726912 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298896690070214, + "loss": 1.7029, + "theoretical_loss": 4.189531190542893, + "tokens_seen": 274857984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629689067201605, + "loss": 1.5462, + "theoretical_loss": 4.189293505722918, + "tokens_seen": 274989056 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294884653961887, + "loss": 1.5461, + "theoretical_loss": 4.189055965871389, + "tokens_seen": 275120128 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.48130497336387634, + "objective/train/avg_token_score": 0.014956628903746605, + "objective/train/docs_used": 167916, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6151095628738403, + "objective/train/original_loss": 3.838587999343872, + "objective/train/theoretical_loss": 4.188818570830883, + "objective/train/tokens_used": 275251200, + "objective/train/unlikelihood_loss": 0.14401310682296753, + "theoretical_loss": 4.188818570830883, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629287863590773, + "loss": 1.5669, + "theoretical_loss": 4.188818570830883, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046290872617853565, + "loss": 1.6388, + "theoretical_loss": 4.188581320444228, + "tokens_seen": 275382272 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462888665997994, + "loss": 1.64, + "theoretical_loss": 4.1883442145544905, + "tokens_seen": 275513344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046286860581745237, + "loss": 1.3469, + "theoretical_loss": 4.188107253004986, + "tokens_seen": 275644416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284854563691073, + "loss": 1.6807, + "theoretical_loss": 4.187870435639275, + "tokens_seen": 275775488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046282848545636915, + "loss": 1.4609, + "theoretical_loss": 4.187633762301159, + "tokens_seen": 275906560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628084252758275, + "loss": 1.3392, + "theoretical_loss": 4.187397232834683, + "tokens_seen": 276037632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627883650952859, + "loss": 1.6093, + "theoretical_loss": 4.1871608470841375, + "tokens_seen": 276168704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046276830491474424, + "loss": 1.6585, + "theoretical_loss": 4.1869246048940525, + "tokens_seen": 276299776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274824473420265, + "loss": 1.6341, + "theoretical_loss": 4.186688506109202, + "tokens_seen": 276430848 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462728184553661, + "loss": 1.513, + "theoretical_loss": 4.186452550574599, + "tokens_seen": 276561920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627081243731194, + "loss": 1.4577, + "theoretical_loss": 4.186216738135501, + "tokens_seen": 276692992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046268806419257774, + "loss": 1.4344, + "theoretical_loss": 4.185981068637401, + "tokens_seen": 276824064 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.46810850501060486, + "objective/train/avg_token_score": 0.0036265684757381678, + "objective/train/docs_used": 168545, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7359020709991455, + "objective/train/original_loss": 3.6703922748565674, + "objective/train/theoretical_loss": 4.185863287443008, + "objective/train/tokens_used": 276889600, + "objective/train/unlikelihood_loss": 0.1548842489719391, + "theoretical_loss": 4.185863287443008, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626680040120361, + "loss": 1.3947, + "theoretical_loss": 4.185745541926035, + "tokens_seen": 276955136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626479438314945, + "loss": 1.4142, + "theoretical_loss": 4.1855101578473795, + "tokens_seen": 277086208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626278836509529, + "loss": 1.5381, + "theoretical_loss": 4.185274916247646, + "tokens_seen": 277217280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260782347041124, + "loss": 1.5502, + "theoretical_loss": 4.185039816973289, + "tokens_seen": 277348352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625877632898696, + "loss": 1.489, + "theoretical_loss": 4.184804859870997, + "tokens_seen": 277479424 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462567703109328, + "loss": 1.4585, + "theoretical_loss": 4.184570044787698, + "tokens_seen": 277610496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625476429287864, + "loss": 1.605, + "theoretical_loss": 4.184335371570556, + "tokens_seen": 277741568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252758274824475, + "loss": 1.5093, + "theoretical_loss": 4.1841008400669715, + "tokens_seen": 277872640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625075225677031, + "loss": 1.6693, + "theoretical_loss": 4.183866450124584, + "tokens_seen": 278003712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046248746238716147, + "loss": 1.5868, + "theoretical_loss": 4.183632201591264, + "tokens_seen": 278134784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624674022066199, + "loss": 1.7203, + "theoretical_loss": 4.1833980943151206, + "tokens_seen": 278265856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244734202607825, + "loss": 1.6068, + "theoretical_loss": 4.183164128144495, + "tokens_seen": 278396928 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.46242669224739075, + "objective/train/avg_token_score": 0.020400462672114372, + "objective/train/docs_used": 169743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8247275352478027, + "objective/train/original_loss": 4.489882946014404, + "objective/train/theoretical_loss": 4.182930302927963, + "objective/train/tokens_used": 278528000, + "objective/train/unlikelihood_loss": 0.11430107802152634, + "theoretical_loss": 4.182930302927963, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624272818455366, + "loss": 1.5359, + "theoretical_loss": 4.182930302927963, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462407221664995, + "loss": 1.7889, + "theoretical_loss": 4.182696618514337, + "tokens_seen": 278659072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623871614844534, + "loss": 1.5107, + "theoretical_loss": 4.1824630747526585, + "tokens_seen": 278790144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236710130391175, + "loss": 1.6843, + "theoretical_loss": 4.182229671492204, + "tokens_seen": 278921216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623470411233701, + "loss": 1.651, + "theoretical_loss": 4.1819964085824815, + "tokens_seen": 279052288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623269809428285, + "loss": 1.5783, + "theoretical_loss": 4.181763285873231, + "tokens_seen": 279183360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230692076228684, + "loss": 1.5391, + "theoretical_loss": 4.181530303214423, + "tokens_seen": 279314432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046228686058174525, + "loss": 1.4712, + "theoretical_loss": 4.181297460456262, + "tokens_seen": 279445504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622668004012036, + "loss": 1.5429, + "theoretical_loss": 4.181064757449178, + "tokens_seen": 279576576 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462246740220662, + "loss": 1.6241, + "theoretical_loss": 4.180832194043836, + "tokens_seen": 279707648 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046222668004012034, + "loss": 1.6594, + "theoretical_loss": 4.180599770091126, + "tokens_seen": 279838720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046220661985957876, + "loss": 1.546, + "theoretical_loss": 4.18036748544217, + "tokens_seen": 279969792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621865596790371, + "loss": 1.4317, + "theoretical_loss": 4.1801353399483165, + "tokens_seen": 280100864 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.34316959977149963, + "objective/train/avg_token_score": 0.019939497113227844, + "objective/train/docs_used": 170343, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2674180269241333, + "objective/train/original_loss": 4.336735725402832, + "objective/train/theoretical_loss": 4.180019319338163, + "objective/train/tokens_used": 280166400, + "objective/train/unlikelihood_loss": 0.17246992886066437, + "theoretical_loss": 4.180019319338163, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621664994984955, + "loss": 1.6044, + "theoretical_loss": 4.179903333461144, + "tokens_seen": 280231936 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214643931795384, + "loss": 1.6516, + "theoretical_loss": 4.179671465832458, + "tokens_seen": 280363008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621263791374122, + "loss": 1.5465, + "theoretical_loss": 4.17943973691429, + "tokens_seen": 280494080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621063189568706, + "loss": 1.5127, + "theoretical_loss": 4.179208146558899, + "tokens_seen": 280625152 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462086258776329, + "loss": 1.5414, + "theoretical_loss": 4.178976694618772, + "tokens_seen": 280756224 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046206619859578735, + "loss": 1.5515, + "theoretical_loss": 4.178745380946619, + "tokens_seen": 280887296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620461384152457, + "loss": 1.4725, + "theoretical_loss": 4.178514205395376, + "tokens_seen": 281018368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620260782347041, + "loss": 1.4888, + "theoretical_loss": 4.178283167818206, + "tokens_seen": 281149440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620060180541625, + "loss": 1.4546, + "theoretical_loss": 4.178052268068494, + "tokens_seen": 281280512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046198595787362085, + "loss": 1.4797, + "theoretical_loss": 4.17782150599985, + "tokens_seen": 281411584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619658976930792, + "loss": 1.4201, + "theoretical_loss": 4.1775908814661085, + "tokens_seen": 281542656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046194583751253763, + "loss": 1.3238, + "theoretical_loss": 4.177360394321325, + "tokens_seen": 281673728 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.38899070024490356, + "objective/train/avg_token_score": 0.016084667295217514, + "objective/train/docs_used": 171008, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.55722177028656, + "objective/train/original_loss": 4.193042278289795, + "objective/train/theoretical_loss": 4.17713004441978, + "objective/train/tokens_used": 281804800, + "objective/train/unlikelihood_loss": 0.173121839761734, + "theoretical_loss": 4.17713004441978, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461925777331996, + "loss": 1.4682, + "theoretical_loss": 4.17713004441978, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046190571715145435, + "loss": 1.4808, + "theoretical_loss": 4.176899831615974, + "tokens_seen": 281935872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618856569709127, + "loss": 1.6193, + "theoretical_loss": 4.176669755764632, + "tokens_seen": 282066944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618655967903711, + "loss": 1.5912, + "theoretical_loss": 4.176439816720697, + "tokens_seen": 282198016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618455366098295, + "loss": 1.5359, + "theoretical_loss": 4.176210014339335, + "tokens_seen": 282329088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046182547642928786, + "loss": 1.6488, + "theoretical_loss": 4.1759803484759335, + "tokens_seen": 282460160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618054162487462, + "loss": 1.354, + "theoretical_loss": 4.175750818986098, + "tokens_seen": 282591232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617853560682046, + "loss": 1.4048, + "theoretical_loss": 4.1755214257256545, + "tokens_seen": 282722304 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461765295887663, + "loss": 1.4483, + "theoretical_loss": 4.175292168550648, + "tokens_seen": 282853376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046174523570712136, + "loss": 1.4696, + "theoretical_loss": 4.175063047317342, + "tokens_seen": 282984448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617251755265797, + "loss": 1.5138, + "theoretical_loss": 4.174834061882218, + "tokens_seen": 283115520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046170511534603814, + "loss": 1.5536, + "theoretical_loss": 4.174605212101977, + "tokens_seen": 283246592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616850551654965, + "loss": 1.3895, + "theoretical_loss": 4.174376497833537, + "tokens_seen": 283377664 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.3909457325935364, + "objective/train/avg_token_score": 0.02921941690146923, + "objective/train/docs_used": 172282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4512357711791992, + "objective/train/original_loss": 4.0204291343688965, + "objective/train/theoretical_loss": 4.174262191471587, + "objective/train/tokens_used": 283443200, + "objective/train/unlikelihood_loss": 0.15194293856620789, + "theoretical_loss": 4.174262191471587, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616649949849549, + "loss": 1.681, + "theoretical_loss": 4.17414791893403, + "tokens_seen": 283508736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616449348044133, + "loss": 1.4106, + "theoretical_loss": 4.173919475260808, + "tokens_seen": 283639808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046162487462387164, + "loss": 1.3723, + "theoretical_loss": 4.173691166671439, + "tokens_seen": 283770880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046160481444333, + "loss": 1.5836, + "theoretical_loss": 4.173462993023706, + "tokens_seen": 283901952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615847542627884, + "loss": 1.6088, + "theoretical_loss": 4.173234954175605, + "tokens_seen": 284033024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615646940822468, + "loss": 1.6808, + "theoretical_loss": 4.173007049985352, + "tokens_seen": 284164096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046154463390170515, + "loss": 1.6575, + "theoretical_loss": 4.172779280311372, + "tokens_seen": 284295168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615245737211635, + "loss": 1.7206, + "theoretical_loss": 4.172551645012307, + "tokens_seen": 284426240 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046150451354062187, + "loss": 1.5265, + "theoretical_loss": 4.172324143947012, + "tokens_seen": 284557312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614844533600803, + "loss": 1.4038, + "theoretical_loss": 4.1720967769745565, + "tokens_seen": 284688384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046146439317953865, + "loss": 1.5643, + "theoretical_loss": 4.1718695439542195, + "tokens_seen": 284819456 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461444332998997, + "loss": 1.6387, + "theoretical_loss": 4.171642444745497, + "tokens_seen": 284950528 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.48081621527671814, + "objective/train/avg_token_score": 0.016248999163508415, + "objective/train/docs_used": 172985, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8190662860870361, + "objective/train/original_loss": 3.9090092182159424, + "objective/train/theoretical_loss": 4.1714154792080915, + "objective/train/tokens_used": 285081600, + "objective/train/unlikelihood_loss": 0.19899842143058777, + "theoretical_loss": 4.1714154792080915, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614242728184554, + "loss": 1.4715, + "theoretical_loss": 4.1714154792080915, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614042126379138, + "loss": 1.5394, + "theoretical_loss": 4.171188647201921, + "tokens_seen": 285212672 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046138415245737215, + "loss": 1.7036, + "theoretical_loss": 4.170961948587115, + "tokens_seen": 285343744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613640922768305, + "loss": 1.5484, + "theoretical_loss": 4.1707353832240095, + "tokens_seen": 285474816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613440320962889, + "loss": 1.5526, + "theoretical_loss": 4.170508950973154, + "tokens_seen": 285605888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046132397191574724, + "loss": 1.5348, + "theoretical_loss": 4.170282651695308, + "tokens_seen": 285736960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046130391173520566, + "loss": 1.5794, + "theoretical_loss": 4.170056485251439, + "tokens_seen": 285868032 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461283851554664, + "loss": 1.4694, + "theoretical_loss": 4.169830451502724, + "tokens_seen": 285999104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612637913741224, + "loss": 1.383, + "theoretical_loss": 4.1696045503105506, + "tokens_seen": 286130176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046124373119358074, + "loss": 1.5947, + "theoretical_loss": 4.169378781536509, + "tokens_seen": 286261248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046122367101303916, + "loss": 1.6029, + "theoretical_loss": 4.169153145042405, + "tokens_seen": 286392320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612036108324975, + "loss": 1.4988, + "theoretical_loss": 4.168927640690246, + "tokens_seen": 286523392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611835506519559, + "loss": 1.4123, + "theoretical_loss": 4.168702268342248, + "tokens_seen": 286654464 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.4003543555736542, + "objective/train/avg_token_score": 0.02369835413992405, + "objective/train/docs_used": 174327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5158040523529053, + "objective/train/original_loss": 4.125824451446533, + "objective/train/theoretical_loss": 4.168589631626808, + "objective/train/tokens_used": 286720000, + "objective/train/unlikelihood_loss": 0.19976606965065002, + "theoretical_loss": 4.168589631626808, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046116349047141425, + "loss": 1.4636, + "theoretical_loss": 4.168477027860833, + "tokens_seen": 286785536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611434302908726, + "loss": 1.6322, + "theoretical_loss": 4.168251919108632, + "tokens_seen": 286916608 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461123370110331, + "loss": 1.4629, + "theoretical_loss": 4.168026941948478, + "tokens_seen": 287047680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611033099297894, + "loss": 1.4635, + "theoretical_loss": 4.1678020962434115, + "tokens_seen": 287178752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046108324974924775, + "loss": 1.2786, + "theoretical_loss": 4.1675773818566775, + "tokens_seen": 287309824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610631895687061, + "loss": 1.5798, + "theoretical_loss": 4.167352798651726, + "tokens_seen": 287440896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610431293881645, + "loss": 1.5423, + "theoretical_loss": 4.167128346492211, + "tokens_seen": 287571968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610230692076229, + "loss": 1.5979, + "theoretical_loss": 4.1669040252419896, + "tokens_seen": 287703040 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046100300902708125, + "loss": 1.3984, + "theoretical_loss": 4.166679834765123, + "tokens_seen": 287834112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609829488465396, + "loss": 1.4598, + "theoretical_loss": 4.166455774925875, + "tokens_seen": 287965184 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046096288866599803, + "loss": 1.6508, + "theoretical_loss": 4.166231845588712, + "tokens_seen": 288096256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609428284854564, + "loss": 1.5655, + "theoretical_loss": 4.166008046618303, + "tokens_seen": 288227328 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.3146994113922119, + "objective/train/avg_token_score": 0.029127070680260658, + "objective/train/docs_used": 174865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2096765041351318, + "objective/train/original_loss": 4.519657611846924, + "objective/train/theoretical_loss": 4.165784377879517, + "objective/train/tokens_used": 288358400, + "objective/train/unlikelihood_loss": 0.12008348852396011, + "theoretical_loss": 4.165784377879517, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046092276830491475, + "loss": 1.6386, + "theoretical_loss": 4.165784377879517, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609027081243731, + "loss": 1.4738, + "theoretical_loss": 4.165560839237429, + "tokens_seen": 288489472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608826479438315, + "loss": 1.553, + "theoretical_loss": 4.16533743055731, + "tokens_seen": 288620544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608625877632899, + "loss": 1.5666, + "theoretical_loss": 4.165114151704634, + "tokens_seen": 288751616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084252758274826, + "loss": 1.2383, + "theoretical_loss": 4.164891002545073, + "tokens_seen": 288882688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608224674022066, + "loss": 1.5886, + "theoretical_loss": 4.164667982944504, + "tokens_seen": 289013760 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460802407221665, + "loss": 1.5561, + "theoretical_loss": 4.1644450927689975, + "tokens_seen": 289144832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607823470411234, + "loss": 1.4849, + "theoretical_loss": 4.164222331884827, + "tokens_seen": 289275904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046076228686058176, + "loss": 1.6219, + "theoretical_loss": 4.163999700158462, + "tokens_seen": 289406976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607422266800401, + "loss": 1.4152, + "theoretical_loss": 4.163777197456573, + "tokens_seen": 289538048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607221664994985, + "loss": 1.6808, + "theoretical_loss": 4.163554823646027, + "tokens_seen": 289669120 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070210631895685, + "loss": 1.4624, + "theoretical_loss": 4.163332578593889, + "tokens_seen": 289800192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046068204613841526, + "loss": 1.42, + "theoretical_loss": 4.1631104621674195, + "tokens_seen": 289931264 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.42473119497299194, + "objective/train/avg_token_score": 0.030998816713690758, + "objective/train/docs_used": 176137, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5777256488800049, + "objective/train/original_loss": 4.12653112411499, + "objective/train/theoretical_loss": 4.162999452147384, + "objective/train/tokens_used": 289996800, + "objective/train/unlikelihood_loss": 0.19045914709568024, + "theoretical_loss": 4.162999452147384, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606619859578736, + "loss": 1.5097, + "theoretical_loss": 4.1628884742340775, + "tokens_seen": 290062336 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460641925777332, + "loss": 1.6269, + "theoretical_loss": 4.162666614661518, + "tokens_seen": 290193408 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046062186559679035, + "loss": 1.5973, + "theoretical_loss": 4.162444883317591, + "tokens_seen": 290324480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060180541624877, + "loss": 1.5025, + "theoretical_loss": 4.162223280070345, + "tokens_seen": 290455552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046058174523570713, + "loss": 1.5917, + "theoretical_loss": 4.162001804788021, + "tokens_seen": 290586624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605616850551655, + "loss": 1.4064, + "theoretical_loss": 4.161780457339055, + "tokens_seen": 290717696 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054162487462385, + "loss": 1.3713, + "theoretical_loss": 4.16155923759208, + "tokens_seen": 290848768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605215646940822, + "loss": 1.4934, + "theoretical_loss": 4.161338145415918, + "tokens_seen": 290979840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050150451354063, + "loss": 1.5155, + "theoretical_loss": 4.161117180679591, + "tokens_seen": 291110912 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460481444332999, + "loss": 1.5663, + "theoretical_loss": 4.160896343252311, + "tokens_seen": 291241984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046138415245736, + "loss": 1.5929, + "theoretical_loss": 4.160675633003484, + "tokens_seen": 291373056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604413239719157, + "loss": 1.4127, + "theoretical_loss": 4.160455049802706, + "tokens_seen": 291504128 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.28060850501060486, + "objective/train/avg_token_score": 0.012403239496052265, + "objective/train/docs_used": 176478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0622302293777466, + "objective/train/original_loss": 3.8111629486083984, + "objective/train/theoretical_loss": 4.160234593519768, + "objective/train/tokens_used": 291635200, + "objective/train/unlikelihood_loss": 0.2176586389541626, + "theoretical_loss": 4.160234593519768, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042126379137414, + "loss": 1.3888, + "theoretical_loss": 4.160234593519768, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604012036108325, + "loss": 1.3968, + "theoretical_loss": 4.160014264024654, + "tokens_seen": 291766272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046038114343029086, + "loss": 1.3323, + "theoretical_loss": 4.159794061187536, + "tokens_seen": 291897344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603610832497492, + "loss": 1.4543, + "theoretical_loss": 4.159573984878779, + "tokens_seen": 292028416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603410230692076, + "loss": 1.5528, + "theoretical_loss": 4.15935403496894, + "tokens_seen": 292159488 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460320962888666, + "loss": 1.4465, + "theoretical_loss": 4.159134211328765, + "tokens_seen": 292290560 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030090270812436, + "loss": 1.5753, + "theoretical_loss": 4.158914513829189, + "tokens_seen": 292421632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602808425275827, + "loss": 1.4256, + "theoretical_loss": 4.158694942341338, + "tokens_seen": 292552704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602607823470411, + "loss": 1.4826, + "theoretical_loss": 4.1584754967365285, + "tokens_seen": 292683776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602407221664995, + "loss": 1.3959, + "theoretical_loss": 4.158256176886264, + "tokens_seen": 292814848 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046022066198595787, + "loss": 1.5701, + "theoretical_loss": 4.158036982662237, + "tokens_seen": 292945920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602006018054163, + "loss": 1.6363, + "theoretical_loss": 4.157817913936329, + "tokens_seen": 293076992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046018054162487465, + "loss": 1.5499, + "theoretical_loss": 4.157598970580608, + "tokens_seen": 293208064 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.3209921717643738, + "objective/train/avg_token_score": 0.019007770344614983, + "objective/train/docs_used": 177825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3222475051879883, + "objective/train/original_loss": 4.558160781860352, + "objective/train/theoretical_loss": 4.157489545876642, + "objective/train/tokens_used": 293273600, + "objective/train/unlikelihood_loss": 0.15462970733642578, + "theoretical_loss": 4.157489545876642, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460160481444333, + "loss": 1.3533, + "theoretical_loss": 4.157380152467333, + "tokens_seen": 293339136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601404212637914, + "loss": 1.6759, + "theoretical_loss": 4.157161459468944, + "tokens_seen": 293470208 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601203610832498, + "loss": 1.6052, + "theoretical_loss": 4.156942891458074, + "tokens_seen": 293601280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046010030090270815, + "loss": 1.5249, + "theoretical_loss": 4.15672444830754, + "tokens_seen": 293732352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600802407221665, + "loss": 1.5622, + "theoretical_loss": 4.156506129890344, + "tokens_seen": 293863424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046006018054162493, + "loss": 1.434, + "theoretical_loss": 4.156287936079675, + "tokens_seen": 293994496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600401203610833, + "loss": 1.516, + "theoretical_loss": 4.156069866748906, + "tokens_seen": 294125568 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046002006018054165, + "loss": 1.3475, + "theoretical_loss": 4.155851921771598, + "tokens_seen": 294256640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 1.6848, + "theoretical_loss": 4.155634101021494, + "tokens_seen": 294387712 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045997993981945843, + "loss": 1.4597, + "theoretical_loss": 4.155416404372522, + "tokens_seen": 294518784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599598796389168, + "loss": 1.538, + "theoretical_loss": 4.155198831698795, + "tokens_seen": 294649856 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993981945837515, + "loss": 1.3701, + "theoretical_loss": 4.154981382874608, + "tokens_seen": 294780928 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.37683284282684326, + "objective/train/avg_token_score": 0.007959270849823952, + "objective/train/docs_used": 179027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.373116374015808, + "objective/train/original_loss": 4.072432518005371, + "objective/train/theoretical_loss": 4.15476405777444, + "objective/train/tokens_used": 294912000, + "objective/train/unlikelihood_loss": 0.14949344098567963, + "theoretical_loss": 4.15476405777444, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599197592778335, + "loss": 1.5478, + "theoretical_loss": 4.15476405777444, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598996990972919, + "loss": 1.4744, + "theoretical_loss": 4.154546856272952, + "tokens_seen": 295043072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598796389167503, + "loss": 1.5242, + "theoretical_loss": 4.154329778244991, + "tokens_seen": 295174144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045985957873620866, + "loss": 1.8468, + "theoretical_loss": 4.154112823565582, + "tokens_seen": 295305216 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459839518555667, + "loss": 1.5245, + "theoretical_loss": 4.153895992109935, + "tokens_seen": 295436288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598194583751254, + "loss": 1.4326, + "theoretical_loss": 4.153679283753439, + "tokens_seen": 295567360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597993981945838, + "loss": 1.5083, + "theoretical_loss": 4.153462698371665, + "tokens_seen": 295698432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045977933801404216, + "loss": 1.6785, + "theoretical_loss": 4.153246235840367, + "tokens_seen": 295829504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597592778335005, + "loss": 1.5052, + "theoretical_loss": 4.153029896035476, + "tokens_seen": 295960576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597392176529589, + "loss": 1.479, + "theoretical_loss": 4.152813678833106, + "tokens_seen": 296091648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045971915747241725, + "loss": 1.5691, + "theoretical_loss": 4.15259758410955, + "tokens_seen": 296222720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969909729187566, + "loss": 1.4449, + "theoretical_loss": 4.152381611741281, + "tokens_seen": 296353792 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459679037111334, + "loss": 1.5655, + "theoretical_loss": 4.152165761604948, + "tokens_seen": 296484864 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.4272971749305725, + "objective/train/avg_token_score": 0.015787431970238686, + "objective/train/docs_used": 179648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6017154455184937, + "objective/train/original_loss": 3.8773791790008545, + "objective/train/theoretical_loss": 4.152057882335261, + "objective/train/tokens_used": 296550400, + "objective/train/unlikelihood_loss": 0.21350431442260742, + "theoretical_loss": 4.152057882335261, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596589769307924, + "loss": 1.7161, + "theoretical_loss": 4.151950033577383, + "tokens_seen": 296615936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045963891675025075, + "loss": 1.5895, + "theoretical_loss": 4.151734427535594, + "tokens_seen": 296747008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045961885656970917, + "loss": 1.4838, + "theoretical_loss": 4.151518943356768, + "tokens_seen": 296878080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045959879638916753, + "loss": 1.6021, + "theoretical_loss": 4.15130358091827, + "tokens_seen": 297009152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595787362086259, + "loss": 1.6677, + "theoretical_loss": 4.151088340097642, + "tokens_seen": 297140224 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045955867602808425, + "loss": 1.5243, + "theoretical_loss": 4.150873220772604, + "tokens_seen": 297271296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595386158475426, + "loss": 1.3541, + "theoretical_loss": 4.1506582228210505, + "tokens_seen": 297402368 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045951855566700103, + "loss": 1.5855, + "theoretical_loss": 4.150443346121054, + "tokens_seen": 297533440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594984954864594, + "loss": 1.5628, + "theoretical_loss": 4.150228590550864, + "tokens_seen": 297664512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045947843530591776, + "loss": 1.3987, + "theoretical_loss": 4.150013955988905, + "tokens_seen": 297795584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594583751253761, + "loss": 1.473, + "theoretical_loss": 4.1497994423137765, + "tokens_seen": 297926656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045943831494483454, + "loss": 1.5786, + "theoretical_loss": 4.149585049404253, + "tokens_seen": 298057728 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.3321114480495453, + "objective/train/avg_token_score": 0.0315096452832222, + "objective/train/docs_used": 180307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3406286239624023, + "objective/train/original_loss": 4.468992710113525, + "objective/train/theoretical_loss": 4.149370777139286, + "objective/train/tokens_used": 298188800, + "objective/train/unlikelihood_loss": 0.12439100444316864, + "theoretical_loss": 4.149370777139286, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594182547642929, + "loss": 1.6037, + "theoretical_loss": 4.149370777139286, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045939819458375126, + "loss": 1.3683, + "theoretical_loss": 4.149156625397998, + "tokens_seen": 298319872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593781344032096, + "loss": 1.4183, + "theoretical_loss": 4.1489425940596885, + "tokens_seen": 298450944 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459358074222668, + "loss": 1.6628, + "theoretical_loss": 4.148728683003829, + "tokens_seen": 298582016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593380140421264, + "loss": 1.4066, + "theoretical_loss": 4.148514892110065, + "tokens_seen": 298713088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931795386158476, + "loss": 1.364, + "theoretical_loss": 4.148301221258217, + "tokens_seen": 298844160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592978936810431, + "loss": 1.5998, + "theoretical_loss": 4.148087670328276, + "tokens_seen": 298975232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592778335005015, + "loss": 1.561, + "theoretical_loss": 4.147874239200405, + "tokens_seen": 299106304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592577733199599, + "loss": 1.5625, + "theoretical_loss": 4.147660927754942, + "tokens_seen": 299237376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923771313941827, + "loss": 1.4573, + "theoretical_loss": 4.147447735872396, + "tokens_seen": 299368448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045921765295887663, + "loss": 1.3989, + "theoretical_loss": 4.147234663433444, + "tokens_seen": 299499520 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459197592778335, + "loss": 1.6051, + "theoretical_loss": 4.14702171031894, + "tokens_seen": 299630592 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045917753259779335, + "loss": 1.4841, + "theoretical_loss": 4.146808876409906, + "tokens_seen": 299761664 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.42644184827804565, + "objective/train/avg_token_score": 0.033878933638334274, + "objective/train/docs_used": 181680, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6177765130996704, + "objective/train/original_loss": 4.189210891723633, + "objective/train/theoretical_loss": 4.146702504120305, + "objective/train/tokens_used": 299827200, + "objective/train/unlikelihood_loss": 0.1963503360748291, + "theoretical_loss": 4.146702504120305, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045915747241725177, + "loss": 1.5309, + "theoretical_loss": 4.146596161587532, + "tokens_seen": 299892736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045913741223671013, + "loss": 1.5576, + "theoretical_loss": 4.146383565733184, + "tokens_seen": 300023808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591173520561685, + "loss": 1.5366, + "theoretical_loss": 4.1461710887283925, + "tokens_seen": 300154880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045909729187562686, + "loss": 1.477, + "theoretical_loss": 4.145958730454861, + "tokens_seen": 300285952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590772316950853, + "loss": 1.4918, + "theoretical_loss": 4.145746490794461, + "tokens_seen": 300417024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045905717151454364, + "loss": 1.5538, + "theoretical_loss": 4.145534369629234, + "tokens_seen": 300548096 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459037111334002, + "loss": 1.5235, + "theoretical_loss": 4.145322366841389, + "tokens_seen": 300679168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045901705115346036, + "loss": 1.3937, + "theoretical_loss": 4.145110482313304, + "tokens_seen": 300810240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589969909729187, + "loss": 1.6506, + "theoretical_loss": 4.144898715927525, + "tokens_seen": 300941312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045897693079237714, + "loss": 1.569, + "theoretical_loss": 4.144687067566765, + "tokens_seen": 301072384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589568706118355, + "loss": 1.4973, + "theoretical_loss": 4.144475537113905, + "tokens_seen": 301203456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045893681043129386, + "loss": 1.4148, + "theoretical_loss": 4.144264124451995, + "tokens_seen": 301334528 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.42748045921325684, + "objective/train/avg_token_score": 0.038286831229925156, + "objective/train/docs_used": 182385, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5852553844451904, + "objective/train/original_loss": 4.035271167755127, + "objective/train/theoretical_loss": 4.144052829464249, + "objective/train/tokens_used": 301465600, + "objective/train/unlikelihood_loss": 0.18358202278614044, + "theoretical_loss": 4.144052829464249, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589167502507522, + "loss": 1.5513, + "theoretical_loss": 4.144052829464249, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045889669007021064, + "loss": 1.6312, + "theoretical_loss": 4.143841652034048, + "tokens_seen": 301596672 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458876629889669, + "loss": 1.5405, + "theoretical_loss": 4.14363059204494, + "tokens_seen": 301727744 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045885656970912737, + "loss": 1.4233, + "theoretical_loss": 4.143419649380639, + "tokens_seen": 301858816 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045883650952858573, + "loss": 1.4781, + "theoretical_loss": 4.143208823925024, + "tokens_seen": 301989888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045881644934804415, + "loss": 1.4335, + "theoretical_loss": 4.142998115562139, + "tokens_seen": 302120960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587963891675025, + "loss": 1.4792, + "theoretical_loss": 4.142787524176194, + "tokens_seen": 302252032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045877632898696087, + "loss": 1.6322, + "theoretical_loss": 4.142577049651563, + "tokens_seen": 302383104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045875626880641923, + "loss": 1.2931, + "theoretical_loss": 4.142366691872784, + "tokens_seen": 302514176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587362086258776, + "loss": 1.5632, + "theoretical_loss": 4.14215645072456, + "tokens_seen": 302645248 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458716148445336, + "loss": 1.6633, + "theoretical_loss": 4.141946326091756, + "tokens_seen": 302776320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586960882647944, + "loss": 1.6206, + "theoretical_loss": 4.1417363178594035, + "tokens_seen": 302907392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586760280842528, + "loss": 1.387, + "theoretical_loss": 4.141526425912694, + "tokens_seen": 303038464 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.2217130959033966, + "objective/train/avg_token_score": 0.04529906064271927, + "objective/train/docs_used": 183646, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8153098225593567, + "objective/train/original_loss": 4.031713485717773, + "objective/train/theoretical_loss": 4.141421523510623, + "objective/train/tokens_used": 303104000, + "objective/train/unlikelihood_loss": 0.2204250991344452, + "theoretical_loss": 4.141421523510623, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045865596790371115, + "loss": 1.3728, + "theoretical_loss": 4.141316650136983, + "tokens_seen": 303169536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045863590772316957, + "loss": 1.4683, + "theoretical_loss": 4.141106990417789, + "tokens_seen": 303300608 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045861584754262793, + "loss": 1.3212, + "theoretical_loss": 4.140897446640793, + "tokens_seen": 303431680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585957873620863, + "loss": 1.4083, + "theoretical_loss": 4.140688018691835, + "tokens_seen": 303562752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045857572718154465, + "loss": 1.4995, + "theoretical_loss": 4.140478706456921, + "tokens_seen": 303693824 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458555667001003, + "loss": 1.5934, + "theoretical_loss": 4.1402695098222155, + "tokens_seen": 303824896 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045853560682046143, + "loss": 1.4326, + "theoretical_loss": 4.140060428674046, + "tokens_seen": 303955968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585155466399198, + "loss": 1.4883, + "theoretical_loss": 4.139851462898897, + "tokens_seen": 304087040 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045849548645937816, + "loss": 1.539, + "theoretical_loss": 4.139642612383418, + "tokens_seen": 304218112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584754262788365, + "loss": 1.5251, + "theoretical_loss": 4.139433877014415, + "tokens_seen": 304349184 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045845536609829494, + "loss": 1.4015, + "theoretical_loss": 4.139225256678857, + "tokens_seen": 304480256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584353059177533, + "loss": 1.7055, + "theoretical_loss": 4.1390167512638705, + "tokens_seen": 304611328 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.5244379043579102, + "objective/train/avg_token_score": 0.007667436730116606, + "objective/train/docs_used": 184075, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9626790285110474, + "objective/train/original_loss": 4.3378424644470215, + "objective/train/theoretical_loss": 4.138808360656742, + "objective/train/tokens_used": 304742400, + "objective/train/unlikelihood_loss": 0.11692871153354645, + "theoretical_loss": 4.138808360656742, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045841524573721166, + "loss": 1.5462, + "theoretical_loss": 4.138808360656742, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045839518555667, + "loss": 1.4083, + "theoretical_loss": 4.138600084744915, + "tokens_seen": 304873472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583751253761284, + "loss": 1.5358, + "theoretical_loss": 4.138391923415996, + "tokens_seen": 305004544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583550651955868, + "loss": 1.5128, + "theoretical_loss": 4.138183876557745, + "tokens_seen": 305135616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045833500501504516, + "loss": 1.6635, + "theoretical_loss": 4.137975944058083, + "tokens_seen": 305266688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583149448345035, + "loss": 1.4921, + "theoretical_loss": 4.1377681258050885, + "tokens_seen": 305397760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582948846539619, + "loss": 1.4856, + "theoretical_loss": 4.137560421686998, + "tokens_seen": 305528832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582748244734203, + "loss": 1.5496, + "theoretical_loss": 4.137352831592203, + "tokens_seen": 305659904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045825476429287867, + "loss": 1.4664, + "theoretical_loss": 4.137145355409253, + "tokens_seen": 305790976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045823470411233703, + "loss": 1.6485, + "theoretical_loss": 4.136937993026857, + "tokens_seen": 305922048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582146439317954, + "loss": 1.5259, + "theoretical_loss": 4.1367307443338746, + "tokens_seen": 306053120 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045819458375125375, + "loss": 1.5475, + "theoretical_loss": 4.136523609219327, + "tokens_seen": 306184192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045817452357071217, + "loss": 1.6011, + "theoretical_loss": 4.136316587572388, + "tokens_seen": 306315264 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.27993646264076233, + "objective/train/avg_token_score": 0.010866036638617516, + "objective/train/docs_used": 185377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9704533815383911, + "objective/train/original_loss": 4.095234394073486, + "objective/train/theoretical_loss": 4.136213119264681, + "objective/train/tokens_used": 306380800, + "objective/train/unlikelihood_loss": 0.20976783335208893, + "theoretical_loss": 4.136213119264681, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045815446339017053, + "loss": 1.4117, + "theoretical_loss": 4.136109679282388, + "tokens_seen": 306446336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581344032096289, + "loss": 1.5928, + "theoretical_loss": 4.135902884238812, + "tokens_seen": 306577408 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045811434302908726, + "loss": 1.6869, + "theoretical_loss": 4.1356962023313, + "tokens_seen": 306708480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580942828485457, + "loss": 1.5054, + "theoretical_loss": 4.135489633449649, + "tokens_seen": 306839552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045807422266800404, + "loss": 1.7384, + "theoretical_loss": 4.135283177483807, + "tokens_seen": 306970624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580541624874624, + "loss": 1.3702, + "theoretical_loss": 4.135076834323876, + "tokens_seen": 307101696 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045803410230692076, + "loss": 1.3647, + "theoretical_loss": 4.134870603860117, + "tokens_seen": 307232768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580140421263791, + "loss": 1.6255, + "theoretical_loss": 4.1346644859829365, + "tokens_seen": 307363840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045799398194583754, + "loss": 1.7572, + "theoretical_loss": 4.134458480582902, + "tokens_seen": 307494912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579739217652959, + "loss": 1.4369, + "theoretical_loss": 4.134252587550728, + "tokens_seen": 307625984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045795386158475426, + "loss": 1.6192, + "theoretical_loss": 4.134046806777286, + "tokens_seen": 307757056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579338014042126, + "loss": 1.6388, + "theoretical_loss": 4.133841138153597, + "tokens_seen": 307888128 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.47434017062187195, + "objective/train/avg_token_score": 0.0074652028270065784, + "objective/train/docs_used": 186163, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8509278297424316, + "objective/train/original_loss": 4.273252487182617, + "objective/train/theoretical_loss": 4.133635581570836, + "objective/train/tokens_used": 308019200, + "objective/train/unlikelihood_loss": 0.12682999670505524, + "theoretical_loss": 4.133635581570836, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045791374122367104, + "loss": 1.7452, + "theoretical_loss": 4.133635581570836, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578936810431294, + "loss": 1.6872, + "theoretical_loss": 4.133430136920327, + "tokens_seen": 308150272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045787362086258777, + "loss": 1.6741, + "theoretical_loss": 4.13322480409355, + "tokens_seen": 308281344 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045785356068204613, + "loss": 1.3866, + "theoretical_loss": 4.133019582982134, + "tokens_seen": 308412416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783350050150455, + "loss": 1.3535, + "theoretical_loss": 4.132814473477857, + "tokens_seen": 308543488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578134403209629, + "loss": 1.7071, + "theoretical_loss": 4.132609475472651, + "tokens_seen": 308674560 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045779338014042127, + "loss": 1.6093, + "theoretical_loss": 4.132404588858597, + "tokens_seen": 308805632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045777331995987963, + "loss": 1.58, + "theoretical_loss": 4.132199813527926, + "tokens_seen": 308936704 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457753259779338, + "loss": 1.4503, + "theoretical_loss": 4.1319951493730205, + "tokens_seen": 309067776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577331995987964, + "loss": 1.6892, + "theoretical_loss": 4.131790596286409, + "tokens_seen": 309198848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577131394182548, + "loss": 1.3993, + "theoretical_loss": 4.131586154160775, + "tokens_seen": 309329920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045769307923771314, + "loss": 1.6309, + "theoretical_loss": 4.131381822888946, + "tokens_seen": 309460992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576730190571715, + "loss": 1.458, + "theoretical_loss": 4.1311776023639, + "tokens_seen": 309592064 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.40237048268318176, + "objective/train/avg_token_score": 0.005976264830678701, + "objective/train/docs_used": 187126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6010853052139282, + "objective/train/original_loss": 4.350051403045654, + "objective/train/theoretical_loss": 4.131075533598018, + "objective/train/tokens_used": 309657600, + "objective/train/unlikelihood_loss": 0.17762263119220734, + "theoretical_loss": 4.131075533598018, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576529588766299, + "loss": 1.6003, + "theoretical_loss": 4.130973492478766, + "tokens_seen": 309723136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576328986960883, + "loss": 1.7154, + "theoretical_loss": 4.130769493126817, + "tokens_seen": 309854208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045761283851554664, + "loss": 1.5257, + "theoretical_loss": 4.130565604201477, + "tokens_seen": 309985280 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457592778335005, + "loss": 1.5008, + "theoretical_loss": 4.130361825596317, + "tokens_seen": 310116352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045757271815446336, + "loss": 1.4277, + "theoretical_loss": 4.130158157205056, + "tokens_seen": 310247424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575526579739218, + "loss": 1.3578, + "theoretical_loss": 4.129954598921559, + "tokens_seen": 310378496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045753259779338014, + "loss": 1.6327, + "theoretical_loss": 4.12975115063984, + "tokens_seen": 310509568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575125376128385, + "loss": 1.726, + "theoretical_loss": 4.1295478122540565, + "tokens_seen": 310640640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045749247743229687, + "loss": 1.7017, + "theoretical_loss": 4.129344583658516, + "tokens_seen": 310771712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574724172517553, + "loss": 1.6805, + "theoretical_loss": 4.1291414647476685, + "tokens_seen": 310902784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045745235707121364, + "loss": 1.5457, + "theoretical_loss": 4.128938455416115, + "tokens_seen": 311033856 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457432296890672, + "loss": 1.592, + "theoretical_loss": 4.128735555558597, + "tokens_seen": 311164928 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.318487286567688, + "objective/train/avg_token_score": 0.01832449808716774, + "objective/train/docs_used": 187875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2443170547485352, + "objective/train/original_loss": 4.4912567138671875, + "objective/train/theoretical_loss": 4.128532765070004, + "objective/train/tokens_used": 311296000, + "objective/train/unlikelihood_loss": 0.08960556238889694, + "theoretical_loss": 4.128532765070004, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045741223671013037, + "loss": 1.4941, + "theoretical_loss": 4.128532765070004, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739217652958873, + "loss": 1.582, + "theoretical_loss": 4.12833008384537, + "tokens_seen": 311427072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045737211634904715, + "loss": 1.7372, + "theoretical_loss": 4.128127511779873, + "tokens_seen": 311558144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573520561685055, + "loss": 1.6319, + "theoretical_loss": 4.127925048768839, + "tokens_seen": 311689216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045733199598796387, + "loss": 1.5441, + "theoretical_loss": 4.1277226947077335, + "tokens_seen": 311820288 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045731193580742223, + "loss": 1.6482, + "theoretical_loss": 4.12752044949217, + "tokens_seen": 311951360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045729187562688065, + "loss": 1.7088, + "theoretical_loss": 4.127318313017904, + "tokens_seen": 312082432 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457271815446339, + "loss": 1.7473, + "theoretical_loss": 4.1271162851808345, + "tokens_seen": 312213504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572517552657974, + "loss": 1.6738, + "theoretical_loss": 4.126914365877004, + "tokens_seen": 312344576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045723169508525574, + "loss": 1.6325, + "theoretical_loss": 4.1267125550026, + "tokens_seen": 312475648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572116349047141, + "loss": 1.5452, + "theoretical_loss": 4.126510852453949, + "tokens_seen": 312606720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045719157472417257, + "loss": 1.634, + "theoretical_loss": 4.126309258127524, + "tokens_seen": 312737792 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045717151454363093, + "loss": 1.3362, + "theoretical_loss": 4.126107771919935, + "tokens_seen": 312868864 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.38135385513305664, + "objective/train/avg_token_score": 0.02425443008542061, + "objective/train/docs_used": 188927, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5612069368362427, + "objective/train/original_loss": 4.165873050689697, + "objective/train/theoretical_loss": 4.126007069328436, + "objective/train/tokens_used": 312934400, + "objective/train/unlikelihood_loss": 0.16727052628993988, + "theoretical_loss": 4.126007069328436, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571514543630893, + "loss": 1.4993, + "theoretical_loss": 4.125906393727941, + "tokens_seen": 312999936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045713139418254766, + "loss": 1.4884, + "theoretical_loss": 4.125705123448437, + "tokens_seen": 313131008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571113340020061, + "loss": 1.6256, + "theoretical_loss": 4.125503960978464, + "tokens_seen": 313262080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045709127382146444, + "loss": 1.6268, + "theoretical_loss": 4.125302906215199, + "tokens_seen": 313393152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570712136409228, + "loss": 1.6244, + "theoretical_loss": 4.125101959055965, + "tokens_seen": 313524224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045705115346038116, + "loss": 1.5808, + "theoretical_loss": 4.124901119398222, + "tokens_seen": 313655296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570310932798395, + "loss": 1.6653, + "theoretical_loss": 4.124700387139574, + "tokens_seen": 313786368 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045701103309929794, + "loss": 1.4647, + "theoretical_loss": 4.124499762177764, + "tokens_seen": 313917440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569909729187563, + "loss": 1.5096, + "theoretical_loss": 4.124299244410672, + "tokens_seen": 314048512 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045697091273821466, + "loss": 1.5286, + "theoretical_loss": 4.124098833736321, + "tokens_seen": 314179584 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456950852557673, + "loss": 1.6552, + "theoretical_loss": 4.123898530052874, + "tokens_seen": 314310656 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045693079237713144, + "loss": 1.5479, + "theoretical_loss": 4.123698333258631, + "tokens_seen": 314441728 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.3544110357761383, + "objective/train/avg_token_score": 0.01743168756365776, + "objective/train/docs_used": 189547, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3702281713485718, + "objective/train/original_loss": 4.191202640533447, + "objective/train/theoretical_loss": 4.123498243252032, + "objective/train/tokens_used": 314572800, + "objective/train/unlikelihood_loss": 0.13637718558311462, + "theoretical_loss": 4.123498243252032, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569107321965898, + "loss": 1.4601, + "theoretical_loss": 4.123498243252032, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045689067201604817, + "loss": 1.5157, + "theoretical_loss": 4.123298259931657, + "tokens_seen": 314703872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045687061183550653, + "loss": 1.5577, + "theoretical_loss": 4.123098383196222, + "tokens_seen": 314834944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045685055165496495, + "loss": 1.7436, + "theoretical_loss": 4.122898612944582, + "tokens_seen": 314966016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568304914744233, + "loss": 1.3226, + "theoretical_loss": 4.122698949075732, + "tokens_seen": 315097088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045681043129388167, + "loss": 1.5545, + "theoretical_loss": 4.1224993914888035, + "tokens_seen": 315228160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045679037111334003, + "loss": 1.5553, + "theoretical_loss": 4.122299940083065, + "tokens_seen": 315359232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567703109327984, + "loss": 1.4723, + "theoretical_loss": 4.122100594757921, + "tokens_seen": 315490304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567502507522568, + "loss": 1.632, + "theoretical_loss": 4.121901355412917, + "tokens_seen": 315621376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567301905717152, + "loss": 1.7226, + "theoretical_loss": 4.121702221947732, + "tokens_seen": 315752448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045671013039117354, + "loss": 1.4859, + "theoretical_loss": 4.121503194262183, + "tokens_seen": 315883520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566900702106319, + "loss": 1.4012, + "theoretical_loss": 4.121304272256222, + "tokens_seen": 316014592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566700100300903, + "loss": 1.474, + "theoretical_loss": 4.121105455829939, + "tokens_seen": 316145664 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.42247068881988525, + "objective/train/avg_token_score": 0.02578432485461235, + "objective/train/docs_used": 190733, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6070023775100708, + "objective/train/original_loss": 4.354072570800781, + "objective/train/theoretical_loss": 4.121006087177992, + "objective/train/tokens_used": 316211200, + "objective/train/unlikelihood_loss": 0.11510311812162399, + "theoretical_loss": 4.121006087177992, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566499498495487, + "loss": 1.6307, + "theoretical_loss": 4.120906744883559, + "tokens_seen": 316276736 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045662988966900704, + "loss": 1.5513, + "theoretical_loss": 4.120708139317441, + "tokens_seen": 316407808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566098294884654, + "loss": 1.6868, + "theoretical_loss": 4.120509639032081, + "tokens_seen": 316538880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045658976930792376, + "loss": 1.5259, + "theoretical_loss": 4.120311243928111, + "tokens_seen": 316669952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565697091273822, + "loss": 1.4202, + "theoretical_loss": 4.120112953906296, + "tokens_seen": 316801024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045654964894684054, + "loss": 1.6459, + "theoretical_loss": 4.119914768867536, + "tokens_seen": 316932096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565295887662989, + "loss": 1.4654, + "theoretical_loss": 4.119716688712866, + "tokens_seen": 317063168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045650952858575727, + "loss": 1.5123, + "theoretical_loss": 4.119518713343455, + "tokens_seen": 317194240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564894684052157, + "loss": 1.5729, + "theoretical_loss": 4.119320842660606, + "tokens_seen": 317325312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045646940822467405, + "loss": 1.5038, + "theoretical_loss": 4.119123076565755, + "tokens_seen": 317456384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564493480441324, + "loss": 1.6548, + "theoretical_loss": 4.118925414960472, + "tokens_seen": 317587456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045642928786359077, + "loss": 1.4868, + "theoretical_loss": 4.11872785774646, + "tokens_seen": 317718528 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5064760446548462, + "objective/train/avg_token_score": 0.0031210684683173895, + "objective/train/docs_used": 191330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9102715253829956, + "objective/train/original_loss": 4.087852954864502, + "objective/train/theoretical_loss": 4.118530404825556, + "objective/train/tokens_used": 317849600, + "objective/train/unlikelihood_loss": 0.16944539546966553, + "theoretical_loss": 4.118530404825556, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045640922768304913, + "loss": 1.4363, + "theoretical_loss": 4.118530404825556, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045638916750250755, + "loss": 1.6041, + "theoretical_loss": 4.118333056099728, + "tokens_seen": 317980672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563691073219659, + "loss": 1.609, + "theoretical_loss": 4.1181358114710775, + "tokens_seen": 318111744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045634904714142427, + "loss": 1.7263, + "theoretical_loss": 4.117938670841838, + "tokens_seen": 318242816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045632898696088263, + "loss": 1.5765, + "theoretical_loss": 4.117741634114376, + "tokens_seen": 318373888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045630892678034105, + "loss": 1.4432, + "theoretical_loss": 4.117544701191187, + "tokens_seen": 318504960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562888665997994, + "loss": 1.7524, + "theoretical_loss": 4.117347871974903, + "tokens_seen": 318636032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562688064192578, + "loss": 1.7409, + "theoretical_loss": 4.117151146368282, + "tokens_seen": 318767104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045624874623871614, + "loss": 1.3192, + "theoretical_loss": 4.116954524274216, + "tokens_seen": 318898176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562286860581745, + "loss": 1.4386, + "theoretical_loss": 4.116758005595727, + "tokens_seen": 319029248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562086258776329, + "loss": 1.462, + "theoretical_loss": 4.116561590235969, + "tokens_seen": 319160320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561885656970913, + "loss": 1.699, + "theoretical_loss": 4.116365278098225, + "tokens_seen": 319291392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045616850551654964, + "loss": 1.4585, + "theoretical_loss": 4.11616906908591, + "tokens_seen": 319422464 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.43585044145584106, + "objective/train/avg_token_score": 0.006533558014780283, + "objective/train/docs_used": 192764, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4908214807510376, + "objective/train/original_loss": 3.6467506885528564, + "objective/train/theoretical_loss": 4.11607100322164, + "objective/train/tokens_used": 319488000, + "objective/train/unlikelihood_loss": 0.22543899714946747, + "theoretical_loss": 4.11607100322164, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456148445336008, + "loss": 1.6432, + "theoretical_loss": 4.115972963102565, + "tokens_seen": 319553536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561283851554664, + "loss": 1.3954, + "theoretical_loss": 4.115776960051864, + "tokens_seen": 319684608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561083249749248, + "loss": 1.4829, + "theoretical_loss": 4.115581059837612, + "tokens_seen": 319815680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045608826479438314, + "loss": 1.6191, + "theoretical_loss": 4.115385262363739, + "tokens_seen": 319946752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560682046138415, + "loss": 1.6366, + "theoretical_loss": 4.115189567534307, + "tokens_seen": 320077824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045604814443329987, + "loss": 1.6404, + "theoretical_loss": 4.114993975253505, + "tokens_seen": 320208896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560280842527583, + "loss": 1.6653, + "theoretical_loss": 4.114798485425652, + "tokens_seen": 320339968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045600802407221665, + "loss": 1.5634, + "theoretical_loss": 4.114603097955197, + "tokens_seen": 320471040 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455987963891675, + "loss": 1.5859, + "theoretical_loss": 4.1144078127467125, + "tokens_seen": 320602112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045596790371113337, + "loss": 1.6533, + "theoretical_loss": 4.114212629704902, + "tokens_seen": 320733184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559478435305918, + "loss": 1.6941, + "theoretical_loss": 4.114017548734598, + "tokens_seen": 320864256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045592778335005015, + "loss": 1.5966, + "theoretical_loss": 4.113822569740757, + "tokens_seen": 320995328 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.3504398763179779, + "objective/train/avg_token_score": 0.021333487704396248, + "objective/train/docs_used": 193449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3244068622589111, + "objective/train/original_loss": 4.0752177238464355, + "objective/train/theoretical_loss": 4.113627692628464, + "objective/train/tokens_used": 321126400, + "objective/train/unlikelihood_loss": 0.22843573987483978, + "theoretical_loss": 4.113627692628464, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559077231695085, + "loss": 1.321, + "theoretical_loss": 4.113627692628464, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558876629889669, + "loss": 1.564, + "theoretical_loss": 4.113432917302934, + "tokens_seen": 321257472 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045586760280842524, + "loss": 1.662, + "theoretical_loss": 4.113238243669504, + "tokens_seen": 321388544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045584754262788365, + "loss": 1.393, + "theoretical_loss": 4.113043671633641, + "tokens_seen": 321519616 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455827482447342, + "loss": 1.4109, + "theoretical_loss": 4.112849201100938, + "tokens_seen": 321650688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558074222668004, + "loss": 1.4717, + "theoretical_loss": 4.112654831977112, + "tokens_seen": 321781760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045578736208625874, + "loss": 1.5382, + "theoretical_loss": 4.112460564168009, + "tokens_seen": 321912832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045576730190571716, + "loss": 1.5993, + "theoretical_loss": 4.112266397579598, + "tokens_seen": 322043904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557472417251755, + "loss": 1.5618, + "theoretical_loss": 4.1120723321179735, + "tokens_seen": 322174976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557271815446339, + "loss": 1.3682, + "theoretical_loss": 4.111878367689359, + "tokens_seen": 322306048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045570712136409224, + "loss": 1.6175, + "theoretical_loss": 4.111684504200099, + "tokens_seen": 322437120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556870611835507, + "loss": 1.605, + "theoretical_loss": 4.111490741556663, + "tokens_seen": 322568192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556670010030091, + "loss": 1.5167, + "theoretical_loss": 4.11129707966565, + "tokens_seen": 322699264 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5134408473968506, + "objective/train/avg_token_score": 0.037201035767793655, + "objective/train/docs_used": 194727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7789528369903564, + "objective/train/original_loss": 3.8756916522979736, + "objective/train/theoretical_loss": 4.111200286473145, + "objective/train/tokens_used": 322764800, + "objective/train/unlikelihood_loss": 0.13461580872535706, + "theoretical_loss": 4.111200286473145, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045564694082246744, + "loss": 1.5212, + "theoretical_loss": 4.111103518433776, + "tokens_seen": 322830336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556268806419258, + "loss": 1.5896, + "theoretical_loss": 4.110910057767887, + "tokens_seen": 322961408 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045560682046138416, + "loss": 1.526, + "theoretical_loss": 4.110716697574951, + "tokens_seen": 323092480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555867602808426, + "loss": 1.458, + "theoretical_loss": 4.110523437762059, + "tokens_seen": 323223552 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045556670010030094, + "loss": 1.4805, + "theoretical_loss": 4.110330278236427, + "tokens_seen": 323354624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555466399197593, + "loss": 1.3518, + "theoretical_loss": 4.110137218905393, + "tokens_seen": 323485696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045552657973921767, + "loss": 1.5839, + "theoretical_loss": 4.109944259676419, + "tokens_seen": 323616768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555065195586761, + "loss": 1.4871, + "theoretical_loss": 4.109751400457089, + "tokens_seen": 323747840 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045548645937813445, + "loss": 1.487, + "theoretical_loss": 4.109558641155112, + "tokens_seen": 323878912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554663991975928, + "loss": 1.5072, + "theoretical_loss": 4.109365981678316, + "tokens_seen": 324009984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045544633901705117, + "loss": 1.4531, + "theoretical_loss": 4.109173421934654, + "tokens_seen": 324141056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045542627883650953, + "loss": 1.5629, + "theoretical_loss": 4.1089809618321995, + "tokens_seen": 324272128 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5034213066101074, + "objective/train/avg_token_score": 0.00968634057790041, + "objective/train/docs_used": 195463, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7959951162338257, + "objective/train/original_loss": 3.7963671684265137, + "objective/train/theoretical_loss": 4.108788601279149, + "objective/train/tokens_used": 324403200, + "objective/train/unlikelihood_loss": 0.1474306285381317, + "theoretical_loss": 4.108788601279149, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045540621865596795, + "loss": 1.5169, + "theoretical_loss": 4.108788601279149, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553861584754263, + "loss": 1.7446, + "theoretical_loss": 4.108596340183819, + "tokens_seen": 324534272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553660982948847, + "loss": 1.6527, + "theoretical_loss": 4.108404178454651, + "tokens_seen": 324665344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045534603811434304, + "loss": 1.5472, + "theoretical_loss": 4.108212116000203, + "tokens_seen": 324796416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045532597793380145, + "loss": 1.6305, + "theoretical_loss": 4.108020152729157, + "tokens_seen": 324927488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553059177532598, + "loss": 1.6545, + "theoretical_loss": 4.107828288550314, + "tokens_seen": 325058560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552858575727182, + "loss": 1.4823, + "theoretical_loss": 4.107636523372598, + "tokens_seen": 325189632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045526579739217654, + "loss": 1.4818, + "theoretical_loss": 4.107444857105052, + "tokens_seen": 325320704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552457372116349, + "loss": 1.6236, + "theoretical_loss": 4.107253289656838, + "tokens_seen": 325451776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552256770310933, + "loss": 1.4742, + "theoretical_loss": 4.10706182093724, + "tokens_seen": 325582848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552056168505517, + "loss": 1.5021, + "theoretical_loss": 4.106870450855661, + "tokens_seen": 325713920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045518555667001004, + "loss": 1.5357, + "theoretical_loss": 4.106679179321622, + "tokens_seen": 325844992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551654964894684, + "loss": 1.7192, + "theoretical_loss": 4.106488006244767, + "tokens_seen": 325976064 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.49883919954299927, + "objective/train/avg_token_score": 0.02902386523783207, + "objective/train/docs_used": 196731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7118357419967651, + "objective/train/original_loss": 3.6418302059173584, + "objective/train/theoretical_loss": 4.106392456599577, + "objective/train/tokens_used": 326041600, + "objective/train/unlikelihood_loss": 0.22379085421562195, + "theoretical_loss": 4.106392456599577, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551454363089268, + "loss": 1.5264, + "theoretical_loss": 4.106296931534854, + "tokens_seen": 326107136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551253761283852, + "loss": 1.5085, + "theoretical_loss": 4.106105955101766, + "tokens_seen": 326238208 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045510531594784354, + "loss": 1.375, + "theoretical_loss": 4.105915076855499, + "tokens_seen": 326369280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550852557673019, + "loss": 1.5322, + "theoretical_loss": 4.105724296706172, + "tokens_seen": 326500352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045506519558676027, + "loss": 1.6069, + "theoretical_loss": 4.1055336145640196, + "tokens_seen": 326631424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550451354062187, + "loss": 1.6159, + "theoretical_loss": 4.105343030339395, + "tokens_seen": 326762496 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045502507522567705, + "loss": 1.5891, + "theoretical_loss": 4.10515254394277, + "tokens_seen": 326893568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550050150451354, + "loss": 1.5234, + "theoretical_loss": 4.104962155284734, + "tokens_seen": 327024640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045498495486459377, + "loss": 1.5651, + "theoretical_loss": 4.104771864275993, + "tokens_seen": 327155712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549648946840522, + "loss": 1.407, + "theoretical_loss": 4.104581670827372, + "tokens_seen": 327286784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045494483450351055, + "loss": 1.6212, + "theoretical_loss": 4.104391574849812, + "tokens_seen": 327417856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549247743229689, + "loss": 1.5187, + "theoretical_loss": 4.104201576254369, + "tokens_seen": 327548928 + }, + { + "debugging/length": 505.3636363636364, + "debugging/num_segments": 11, + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5214442610740662, + "objective/train/avg_token_score": 0.01770769990980625, + "objective/train/docs_used": 197513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0139143466949463, + "objective/train/original_loss": 4.308063983917236, + "objective/train/theoretical_loss": 4.10401167495222, + "objective/train/tokens_used": 327680000, + "objective/train/unlikelihood_loss": 0.12178285419940948, + "theoretical_loss": 4.10401167495222, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549047141424273, + "loss": 1.6542, + "theoretical_loss": 4.10401167495222, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045488465396188564, + "loss": 1.6045, + "theoretical_loss": 4.103821870854656, + "tokens_seen": 327811072 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045486459378134405, + "loss": 1.5895, + "theoretical_loss": 4.103632163873083, + "tokens_seen": 327942144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548445336008024, + "loss": 1.6359, + "theoretical_loss": 4.103442553919026, + "tokens_seen": 328073216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548244734202608, + "loss": 1.6449, + "theoretical_loss": 4.103253040904124, + "tokens_seen": 328204288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045480441323971914, + "loss": 1.5452, + "theoretical_loss": 4.103063624740133, + "tokens_seen": 328335360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045478435305917756, + "loss": 1.4296, + "theoretical_loss": 4.102874305338923, + "tokens_seen": 328466432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547642928786359, + "loss": 1.532, + "theoretical_loss": 4.10268508261248, + "tokens_seen": 328597504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547442326980943, + "loss": 1.6441, + "theoretical_loss": 4.1024959564729055, + "tokens_seen": 328728576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045472417251755264, + "loss": 1.5003, + "theoretical_loss": 4.102306926832417, + "tokens_seen": 328859648 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454704112337011, + "loss": 1.5285, + "theoretical_loss": 4.1021179936033425, + "tokens_seen": 328990720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546840521564694, + "loss": 1.5756, + "theoretical_loss": 4.10192915669813, + "tokens_seen": 329121792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546639919759278, + "loss": 1.5778, + "theoretical_loss": 4.101740416029338, + "tokens_seen": 329252864 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.48374876379966736, + "objective/train/avg_token_score": 0.012546883895993233, + "objective/train/docs_used": 198138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9386160373687744, + "objective/train/original_loss": 4.203801155090332, + "objective/train/theoretical_loss": 4.101646081756305, + "objective/train/tokens_used": 329318400, + "objective/train/unlikelihood_loss": 0.1514439433813095, + "theoretical_loss": 4.101646081756305, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045464393179538615, + "loss": 1.4654, + "theoretical_loss": 4.101551771509641, + "tokens_seen": 329383936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546238716148445, + "loss": 1.6519, + "theoretical_loss": 4.101363223051826, + "tokens_seen": 329515008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546038114343029, + "loss": 1.4715, + "theoretical_loss": 4.101174770568795, + "tokens_seen": 329646080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545837512537613, + "loss": 1.4744, + "theoretical_loss": 4.100986413973564, + "tokens_seen": 329777152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045456369107321965, + "loss": 1.7026, + "theoretical_loss": 4.1007981531792606, + "tokens_seen": 329908224 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454543630892678, + "loss": 1.4468, + "theoretical_loss": 4.1006099880991265, + "tokens_seen": 330039296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045452357071213643, + "loss": 1.5817, + "theoretical_loss": 4.100421918646517, + "tokens_seen": 330170368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545035105315948, + "loss": 1.5614, + "theoretical_loss": 4.100233944734899, + "tokens_seen": 330301440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045448345035105315, + "loss": 1.6208, + "theoretical_loss": 4.100046066277853, + "tokens_seen": 330432512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544633901705115, + "loss": 1.5886, + "theoretical_loss": 4.0998582831890715, + "tokens_seen": 330563584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544433299899699, + "loss": 1.5603, + "theoretical_loss": 4.09967059538236, + "tokens_seen": 330694656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544232698094283, + "loss": 1.5664, + "theoretical_loss": 4.099483002771633, + "tokens_seen": 330825728 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.34042033553123474, + "objective/train/avg_token_score": 0.018601443618535995, + "objective/train/docs_used": 199416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2680652141571045, + "objective/train/original_loss": 3.715749740600586, + "objective/train/theoretical_loss": 4.099295505270921, + "objective/train/tokens_used": 330956800, + "objective/train/unlikelihood_loss": 0.23622927069664001, + "theoretical_loss": 4.099295505270921, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045440320962888666, + "loss": 1.3597, + "theoretical_loss": 4.099295505270921, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454383149448345, + "loss": 1.4881, + "theoretical_loss": 4.099108102794363, + "tokens_seen": 331087872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543630892678034, + "loss": 1.5775, + "theoretical_loss": 4.098920795256213, + "tokens_seen": 331218944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543430290872618, + "loss": 1.6801, + "theoretical_loss": 4.098733582570831, + "tokens_seen": 331350016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045432296890672016, + "loss": 1.6044, + "theoretical_loss": 4.098546464652693, + "tokens_seen": 331481088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543029087261785, + "loss": 1.5239, + "theoretical_loss": 4.098359441416383, + "tokens_seen": 331612160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542828485456369, + "loss": 1.7179, + "theoretical_loss": 4.098172512776597, + "tokens_seen": 331743232 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045426278836509525, + "loss": 1.5754, + "theoretical_loss": 4.097985678648142, + "tokens_seen": 331874304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045424272818455366, + "loss": 1.5853, + "theoretical_loss": 4.097798938945933, + "tokens_seen": 332005376 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454222668004012, + "loss": 1.4991, + "theoretical_loss": 4.097612293584998, + "tokens_seen": 332136448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045420260782347044, + "loss": 1.389, + "theoretical_loss": 4.097425742480472, + "tokens_seen": 332267520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541825476429288, + "loss": 1.7117, + "theoretical_loss": 4.0972392855476025, + "tokens_seen": 332398592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541624874623872, + "loss": 1.5459, + "theoretical_loss": 4.097052922701746, + "tokens_seen": 332529664 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.3013196587562561, + "objective/train/avg_token_score": 0.018458493053913116, + "objective/train/docs_used": 200524, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0405497550964355, + "objective/train/original_loss": 3.868088960647583, + "objective/train/theoretical_loss": 4.0969597765350265, + "objective/train/tokens_used": 332595200, + "objective/train/unlikelihood_loss": 0.16207247972488403, + "theoretical_loss": 4.0969597765350265, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541424272818456, + "loss": 1.367, + "theoretical_loss": 4.096866653858368, + "tokens_seen": 332660736 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045412236710130395, + "loss": 1.617, + "theoretical_loss": 4.096680478933042, + "tokens_seen": 332791808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541023069207623, + "loss": 1.5846, + "theoretical_loss": 4.09649439784145, + "tokens_seen": 332922880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045408224674022067, + "loss": 1.6318, + "theoretical_loss": 4.09630841049939, + "tokens_seen": 333053952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540621865596791, + "loss": 1.655, + "theoretical_loss": 4.096122516822757, + "tokens_seen": 333185024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045404212637913745, + "loss": 1.4591, + "theoretical_loss": 4.095936716727564, + "tokens_seen": 333316096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540220661985958, + "loss": 1.4523, + "theoretical_loss": 4.095751010129929, + "tokens_seen": 333447168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045400200601805417, + "loss": 1.6392, + "theoretical_loss": 4.0955653969460775, + "tokens_seen": 333578240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539819458375126, + "loss": 1.5289, + "theoretical_loss": 4.095379877092343, + "tokens_seen": 333709312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045396188565697095, + "loss": 1.7503, + "theoretical_loss": 4.0951944504851685, + "tokens_seen": 333840384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539418254764293, + "loss": 1.4979, + "theoretical_loss": 4.095009117041102, + "tokens_seen": 333971456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539217652958877, + "loss": 1.7049, + "theoretical_loss": 4.094823876676802, + "tokens_seen": 334102528 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5921309590339661, + "objective/train/avg_token_score": 0.006544371135532856, + "objective/train/docs_used": 201196, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.2652223110198975, + "objective/train/original_loss": 4.068779468536377, + "objective/train/theoretical_loss": 4.094638729309031, + "objective/train/tokens_used": 334233600, + "objective/train/unlikelihood_loss": 0.11625197529792786, + "theoretical_loss": 4.094638729309031, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045390170511534604, + "loss": 1.4098, + "theoretical_loss": 4.094638729309031, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045388164493480445, + "loss": 1.6587, + "theoretical_loss": 4.09445367485466, + "tokens_seen": 334364672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538615847542628, + "loss": 1.5036, + "theoretical_loss": 4.094268713230667, + "tokens_seen": 334495744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538415245737212, + "loss": 1.4698, + "theoretical_loss": 4.094083844354137, + "tokens_seen": 334626816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045382146439317954, + "loss": 1.4929, + "theoretical_loss": 4.093899068142262, + "tokens_seen": 334757888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045380140421263796, + "loss": 1.6098, + "theoretical_loss": 4.093714384512337, + "tokens_seen": 334888960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537813440320963, + "loss": 1.6043, + "theoretical_loss": 4.093529793381768, + "tokens_seen": 335020032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537612838515547, + "loss": 1.6377, + "theoretical_loss": 4.093345294668063, + "tokens_seen": 335151104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045374122367101304, + "loss": 1.7649, + "theoretical_loss": 4.093160888288837, + "tokens_seen": 335282176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537211634904714, + "loss": 1.5487, + "theoretical_loss": 4.0929765741618125, + "tokens_seen": 335413248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537011033099298, + "loss": 1.519, + "theoretical_loss": 4.092792352204814, + "tokens_seen": 335544320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536810431293882, + "loss": 1.5532, + "theoretical_loss": 4.092608222335774, + "tokens_seen": 335675392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045366098294884655, + "loss": 1.5037, + "theoretical_loss": 4.09242418447273, + "tokens_seen": 335806464 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.4757453501224518, + "objective/train/avg_token_score": 0.009879210032522678, + "objective/train/docs_used": 202460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8038827180862427, + "objective/train/original_loss": 4.1414971351623535, + "objective/train/theoretical_loss": 4.092332200017871, + "objective/train/tokens_used": 335872000, + "objective/train/unlikelihood_loss": 0.12569139897823334, + "theoretical_loss": 4.092332200017871, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536409227683049, + "loss": 1.7663, + "theoretical_loss": 4.092240238533822, + "tokens_seen": 335937536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536208625877633, + "loss": 1.5856, + "theoretical_loss": 4.092056384437297, + "tokens_seen": 336068608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536008024072217, + "loss": 1.63, + "theoretical_loss": 4.091872622101506, + "tokens_seen": 336199680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045358074222668005, + "loss": 1.6248, + "theoretical_loss": 4.091688951444904, + "tokens_seen": 336330752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535606820461384, + "loss": 1.5825, + "theoretical_loss": 4.091505372386051, + "tokens_seen": 336461824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045354062186559683, + "loss": 1.6858, + "theoretical_loss": 4.091321884843609, + "tokens_seen": 336592896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535205616850552, + "loss": 1.5953, + "theoretical_loss": 4.0911384887363464, + "tokens_seen": 336723968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045350050150451355, + "loss": 1.6498, + "theoretical_loss": 4.0909551839831355, + "tokens_seen": 336855040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534804413239719, + "loss": 1.5834, + "theoretical_loss": 4.090771970502948, + "tokens_seen": 336986112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534603811434303, + "loss": 1.5166, + "theoretical_loss": 4.090588848214865, + "tokens_seen": 337117184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534403209628887, + "loss": 1.5447, + "theoretical_loss": 4.090405817038065, + "tokens_seen": 337248256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045342026078234706, + "loss": 1.4795, + "theoretical_loss": 4.090222876891834, + "tokens_seen": 337379328 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.4949902296066284, + "objective/train/avg_token_score": 0.009650841355323792, + "objective/train/docs_used": 203033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7750602960586548, + "objective/train/original_loss": 3.9335618019104004, + "objective/train/theoretical_loss": 4.090040027695556, + "objective/train/tokens_used": 337510400, + "objective/train/unlikelihood_loss": 0.13872487843036652, + "theoretical_loss": 4.090040027695556, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534002006018054, + "loss": 1.712, + "theoretical_loss": 4.090040027695556, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533801404212638, + "loss": 1.874, + "theoretical_loss": 4.089857269368725, + "tokens_seen": 337641472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533600802407222, + "loss": 1.5283, + "theoretical_loss": 4.089674601830929, + "tokens_seen": 337772544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045334002006018056, + "loss": 1.6553, + "theoretical_loss": 4.089492025001864, + "tokens_seen": 337903616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533199598796389, + "loss": 1.5234, + "theoretical_loss": 4.089309538801327, + "tokens_seen": 338034688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532998996990973, + "loss": 1.5957, + "theoretical_loss": 4.0891271431492155, + "tokens_seen": 338165760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045327983951855565, + "loss": 1.7097, + "theoretical_loss": 4.08894483796553, + "tokens_seen": 338296832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045325977933801406, + "loss": 1.6363, + "theoretical_loss": 4.088762623170373, + "tokens_seen": 338427904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532397191574724, + "loss": 1.6686, + "theoretical_loss": 4.088580498683948, + "tokens_seen": 338558976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532196589769308, + "loss": 1.6399, + "theoretical_loss": 4.088398464426559, + "tokens_seen": 338690048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045319959879638915, + "loss": 1.6365, + "theoretical_loss": 4.088216520318612, + "tokens_seen": 338821120 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045317953861584757, + "loss": 1.5666, + "theoretical_loss": 4.088034666280614, + "tokens_seen": 338952192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045315947843530593, + "loss": 1.6203, + "theoretical_loss": 4.087852902233173, + "tokens_seen": 339083264 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.4335288405418396, + "objective/train/avg_token_score": 0.013492316007614136, + "objective/train/docs_used": 204232, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5629843473434448, + "objective/train/original_loss": 3.536388874053955, + "objective/train/theoretical_loss": 4.087762053931129, + "objective/train/tokens_used": 339148800, + "objective/train/unlikelihood_loss": 0.25973203778266907, + "theoretical_loss": 4.087762053931129, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531394182547643, + "loss": 1.6724, + "theoretical_loss": 4.087671228096997, + "tokens_seen": 339214336 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045311935807422265, + "loss": 1.5495, + "theoretical_loss": 4.087489643792894, + "tokens_seen": 339345408 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453099297893681, + "loss": 1.6182, + "theoretical_loss": 4.087308149241774, + "tokens_seen": 339476480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045307923771313943, + "loss": 1.6192, + "theoretical_loss": 4.087126744364646, + "tokens_seen": 339607552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530591775325978, + "loss": 1.5072, + "theoretical_loss": 4.086945429082618, + "tokens_seen": 339738624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045303911735205616, + "loss": 1.6224, + "theoretical_loss": 4.086764203316902, + "tokens_seen": 339869696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530190571715145, + "loss": 1.5255, + "theoretical_loss": 4.086583066988802, + "tokens_seen": 340000768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045299899699097294, + "loss": 1.5888, + "theoretical_loss": 4.0864020200197295, + "tokens_seen": 340131840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529789368104313, + "loss": 1.6588, + "theoretical_loss": 4.086221062331192, + "tokens_seen": 340262912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045295887662988966, + "loss": 1.6366, + "theoretical_loss": 4.086040193844794, + "tokens_seen": 340393984 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452938816449348, + "loss": 1.4848, + "theoretical_loss": 4.0858594144822415, + "tokens_seen": 340525056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529187562688064, + "loss": 1.5903, + "theoretical_loss": 4.085678724165341, + "tokens_seen": 340656128 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.32972875237464905, + "objective/train/avg_token_score": 0.018639761954545975, + "objective/train/docs_used": 205032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0958448648452759, + "objective/train/original_loss": 3.859755754470825, + "objective/train/theoretical_loss": 4.085498122815992, + "objective/train/tokens_used": 340787200, + "objective/train/unlikelihood_loss": 0.2070046067237854, + "theoretical_loss": 4.085498122815992, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528986960882648, + "loss": 1.5739, + "theoretical_loss": 4.085498122815992, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045287863590772316, + "loss": 1.6326, + "theoretical_loss": 4.085317610356199, + "tokens_seen": 340918272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528585757271815, + "loss": 1.4208, + "theoretical_loss": 4.08513718670806, + "tokens_seen": 341049344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528385155466399, + "loss": 1.5432, + "theoretical_loss": 4.084956851793773, + "tokens_seen": 341180416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528184553660983, + "loss": 1.5042, + "theoretical_loss": 4.0847766055356365, + "tokens_seen": 341311488 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045279839518555667, + "loss": 1.6324, + "theoretical_loss": 4.0845964478560415, + "tokens_seen": 341442560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045277833500501503, + "loss": 1.4951, + "theoretical_loss": 4.0844163786774805, + "tokens_seen": 341573632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527582748244734, + "loss": 1.6173, + "theoretical_loss": 4.084236397922544, + "tokens_seen": 341704704 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045273821464393175, + "loss": 1.5795, + "theoretical_loss": 4.0840565055139155, + "tokens_seen": 341835776 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045271815446339017, + "loss": 1.5769, + "theoretical_loss": 4.083876701374381, + "tokens_seen": 341966848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526980942828486, + "loss": 1.505, + "theoretical_loss": 4.08369698542682, + "tokens_seen": 342097920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045267803410230695, + "loss": 1.5581, + "theoretical_loss": 4.083517357594212, + "tokens_seen": 342228992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526579739217653, + "loss": 1.3931, + "theoretical_loss": 4.083337817799629, + "tokens_seen": 342360064 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.339259535074234, + "objective/train/avg_token_score": 0.027388932183384895, + "objective/train/docs_used": 206109, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2726820707321167, + "objective/train/original_loss": 4.560004234313965, + "objective/train/theoretical_loss": 4.083248080892584, + "objective/train/tokens_used": 342425600, + "objective/train/unlikelihood_loss": 0.11201095581054688, + "theoretical_loss": 4.083248080892584, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526379137412237, + "loss": 1.5391, + "theoretical_loss": 4.083158365966243, + "tokens_seen": 342491136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526178535606821, + "loss": 1.6968, + "theoretical_loss": 4.082979002017321, + "tokens_seen": 342622208 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045259779338014045, + "loss": 1.6573, + "theoretical_loss": 4.082799725876228, + "tokens_seen": 342753280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525777331995988, + "loss": 1.6508, + "theoretical_loss": 4.082620537466422, + "tokens_seen": 342884352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045255767301905723, + "loss": 1.6432, + "theoretical_loss": 4.082441436711459, + "tokens_seen": 343015424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525376128385156, + "loss": 1.5502, + "theoretical_loss": 4.082262423534992, + "tokens_seen": 343146496 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045251755265797395, + "loss": 1.4503, + "theoretical_loss": 4.082083497860766, + "tokens_seen": 343277568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524974924774323, + "loss": 1.5342, + "theoretical_loss": 4.081904659612624, + "tokens_seen": 343408640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524774322968907, + "loss": 1.3966, + "theoretical_loss": 4.081725908714505, + "tokens_seen": 343539712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524573721163491, + "loss": 1.4216, + "theoretical_loss": 4.08154724509044, + "tokens_seen": 343670784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045243731193580746, + "loss": 1.3198, + "theoretical_loss": 4.08136866866456, + "tokens_seen": 343801856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524172517552658, + "loss": 1.5836, + "theoretical_loss": 4.0811901793610845, + "tokens_seen": 343932928 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.34237536787986755, + "objective/train/avg_token_score": 0.0033272351138293743, + "objective/train/docs_used": 206760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2112702131271362, + "objective/train/original_loss": 3.6624834537506104, + "objective/train/theoretical_loss": 4.081011777104333, + "objective/train/tokens_used": 344064000, + "objective/train/unlikelihood_loss": 0.18153946101665497, + "theoretical_loss": 4.081011777104333, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523971915747242, + "loss": 1.4097, + "theoretical_loss": 4.081011777104333, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523771313941826, + "loss": 1.7856, + "theoretical_loss": 4.080833461818718, + "tokens_seen": 344195072 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045235707121364096, + "loss": 1.5064, + "theoretical_loss": 4.080655233428745, + "tokens_seen": 344326144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523370110330993, + "loss": 1.5193, + "theoretical_loss": 4.080477091859015, + "tokens_seen": 344457216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523169508525577, + "loss": 1.5518, + "theoretical_loss": 4.080299037034224, + "tokens_seen": 344588288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045229689067201605, + "loss": 1.6474, + "theoretical_loss": 4.0801210688791585, + "tokens_seen": 344719360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045227683049147446, + "loss": 1.5025, + "theoretical_loss": 4.079943187318703, + "tokens_seen": 344850432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522567703109328, + "loss": 1.6411, + "theoretical_loss": 4.0797653922778325, + "tokens_seen": 344981504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522367101303912, + "loss": 1.6179, + "theoretical_loss": 4.079587683681617, + "tokens_seen": 345112576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045221664994984955, + "loss": 1.4868, + "theoretical_loss": 4.07941006145522, + "tokens_seen": 345243648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045219658976930797, + "loss": 1.357, + "theoretical_loss": 4.0792325255238975, + "tokens_seen": 345374720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045217652958876633, + "loss": 1.4853, + "theoretical_loss": 4.079055075812997, + "tokens_seen": 345505792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521564694082247, + "loss": 1.6133, + "theoretical_loss": 4.078877712247963, + "tokens_seen": 345636864 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.1969086080789566, + "objective/train/avg_token_score": 0.01607353612780571, + "objective/train/docs_used": 207336, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7453041076660156, + "objective/train/original_loss": 4.304450988769531, + "objective/train/theoretical_loss": 4.078789062746872, + "objective/train/tokens_used": 345702400, + "objective/train/unlikelihood_loss": 0.1982918530702591, + "theoretical_loss": 4.078789062746872, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045213640922768305, + "loss": 1.3566, + "theoretical_loss": 4.07870043475433, + "tokens_seen": 345767936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521163490471414, + "loss": 1.4831, + "theoretical_loss": 4.078523243257723, + "tokens_seen": 345899008 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045209628886659983, + "loss": 1.6821, + "theoretical_loss": 4.078346137683862, + "tokens_seen": 346030080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520762286860582, + "loss": 1.5421, + "theoretical_loss": 4.078169117958562, + "tokens_seen": 346161152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045205616850551656, + "loss": 1.4795, + "theoretical_loss": 4.077992184007725, + "tokens_seen": 346292224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520361083249749, + "loss": 1.6154, + "theoretical_loss": 4.077815335757347, + "tokens_seen": 346423296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045201604814443334, + "loss": 1.5112, + "theoretical_loss": 4.077638573133515, + "tokens_seen": 346554368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519959879638917, + "loss": 1.5462, + "theoretical_loss": 4.0774618960624105, + "tokens_seen": 346685440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045197592778335006, + "loss": 1.6378, + "theoretical_loss": 4.077285304470304, + "tokens_seen": 346816512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519558676028084, + "loss": 1.5589, + "theoretical_loss": 4.077108798283557, + "tokens_seen": 346947584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519358074222668, + "loss": 1.3784, + "theoretical_loss": 4.076932377428625, + "tokens_seen": 347078656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519157472417252, + "loss": 1.4946, + "theoretical_loss": 4.076756041832049, + "tokens_seen": 347209728 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3781769275665283, + "objective/train/avg_token_score": 0.007616470567882061, + "objective/train/docs_used": 208796, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4440606832504272, + "objective/train/original_loss": 3.906726360321045, + "objective/train/theoretical_loss": 4.076579791420469, + "objective/train/tokens_used": 347340800, + "objective/train/unlikelihood_loss": 0.1508537232875824, + "theoretical_loss": 4.076579791420469, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045189568706118356, + "loss": 1.6182, + "theoretical_loss": 4.076579791420469, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518756268806419, + "loss": 1.5178, + "theoretical_loss": 4.07640362612061, + "tokens_seen": 347471872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518555667001003, + "loss": 1.529, + "theoretical_loss": 4.076227545859288, + "tokens_seen": 347602944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518355065195587, + "loss": 1.6502, + "theoretical_loss": 4.076051550563411, + "tokens_seen": 347734016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045181544633901707, + "loss": 1.3819, + "theoretical_loss": 4.075875640159978, + "tokens_seen": 347865088 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045179538615847543, + "loss": 1.5274, + "theoretical_loss": 4.075699814576076, + "tokens_seen": 347996160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517753259779338, + "loss": 1.5282, + "theoretical_loss": 4.075524073738883, + "tokens_seen": 348127232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045175526579739215, + "loss": 1.5711, + "theoretical_loss": 4.075348417575669, + "tokens_seen": 348258304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173520561685057, + "loss": 1.6626, + "theoretical_loss": 4.075172846013789, + "tokens_seen": 348389376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045171514543630893, + "loss": 1.4903, + "theoretical_loss": 4.074997358980694, + "tokens_seen": 348520448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516950852557673, + "loss": 1.6022, + "theoretical_loss": 4.074821956403917, + "tokens_seen": 348651520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045167502507522566, + "loss": 1.5102, + "theoretical_loss": 4.074646638211087, + "tokens_seen": 348782592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045165496489468407, + "loss": 1.6114, + "theoretical_loss": 4.074471404329918, + "tokens_seen": 348913664 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5323802828788757, + "objective/train/avg_token_score": 0.004500185139477253, + "objective/train/docs_used": 209597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1040139198303223, + "objective/train/original_loss": 4.173861980438232, + "objective/train/theoretical_loss": 4.074383818983643, + "objective/train/tokens_used": 348979200, + "objective/train/unlikelihood_loss": 0.11532780528068542, + "theoretical_loss": 4.074383818983643, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045163490471414244, + "loss": 1.7397, + "theoretical_loss": 4.074296254688216, + "tokens_seen": 349044736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516148445336008, + "loss": 1.5713, + "theoretical_loss": 4.074121189213873, + "tokens_seen": 349175808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045159478435305916, + "loss": 1.4462, + "theoretical_loss": 4.073946207834872, + "tokens_seen": 349306880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515747241725175, + "loss": 1.5493, + "theoretical_loss": 4.073771310479284, + "tokens_seen": 349437952 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045155466399197594, + "loss": 1.4127, + "theoretical_loss": 4.073596497075267, + "tokens_seen": 349569024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515346038114343, + "loss": 1.3747, + "theoretical_loss": 4.07342176755107, + "tokens_seen": 349700096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045151454363089266, + "loss": 1.4241, + "theoretical_loss": 4.073247121835028, + "tokens_seen": 349831168 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451494483450351, + "loss": 1.5795, + "theoretical_loss": 4.073072559855565, + "tokens_seen": 349962240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045147442326980944, + "loss": 1.5063, + "theoretical_loss": 4.072898081541193, + "tokens_seen": 350093312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514543630892678, + "loss": 1.4867, + "theoretical_loss": 4.0727236868205114, + "tokens_seen": 350224384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045143430290872617, + "loss": 1.4653, + "theoretical_loss": 4.072549375622207, + "tokens_seen": 350355456 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045141424272818453, + "loss": 1.5842, + "theoretical_loss": 4.072375147875054, + "tokens_seen": 350486528 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3748167157173157, + "objective/train/avg_token_score": 0.026631662622094154, + "objective/train/docs_used": 210243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2623140811920166, + "objective/train/original_loss": 4.159633636474609, + "objective/train/theoretical_loss": 4.0722010035079155, + "objective/train/tokens_used": 350617600, + "objective/train/unlikelihood_loss": 0.13227179646492004, + "theoretical_loss": 4.0722010035079155, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045139418254764294, + "loss": 1.4965, + "theoretical_loss": 4.0722010035079155, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513741223671013, + "loss": 1.4648, + "theoretical_loss": 4.07202694244974, + "tokens_seen": 350748672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045135406218655967, + "loss": 1.2912, + "theoretical_loss": 4.0718529646295645, + "tokens_seen": 350879744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045133400200601803, + "loss": 1.3967, + "theoretical_loss": 4.071679069976511, + "tokens_seen": 351010816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513139418254764, + "loss": 1.4605, + "theoretical_loss": 4.07150525841979, + "tokens_seen": 351141888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512938816449348, + "loss": 1.4834, + "theoretical_loss": 4.071331529888698, + "tokens_seen": 351272960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045127382146439317, + "loss": 1.4192, + "theoretical_loss": 4.071157884312619, + "tokens_seen": 351404032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125376128385153, + "loss": 1.5059, + "theoretical_loss": 4.070984321621021, + "tokens_seen": 351535104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512337011033099, + "loss": 1.5377, + "theoretical_loss": 4.0708108417434605, + "tokens_seen": 351666176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512136409227683, + "loss": 1.4409, + "theoretical_loss": 4.07063744460958, + "tokens_seen": 351797248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045119358074222673, + "loss": 1.4988, + "theoretical_loss": 4.070464130149106, + "tokens_seen": 351928320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511735205616851, + "loss": 1.5584, + "theoretical_loss": 4.070290898291852, + "tokens_seen": 352059392 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045115346038114345, + "loss": 1.5342, + "theoretical_loss": 4.07011774896772, + "tokens_seen": 352190464 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5329301357269287, + "objective/train/avg_token_score": 0.007351990323513746, + "objective/train/docs_used": 211560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9991000890731812, + "objective/train/original_loss": 4.122071743011475, + "objective/train/theoretical_loss": 4.07003120523369, + "objective/train/tokens_used": 352256000, + "objective/train/unlikelihood_loss": 0.16980284452438354, + "theoretical_loss": 4.07003120523369, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511334002006018, + "loss": 1.599, + "theoretical_loss": 4.069944682106691, + "tokens_seen": 352321536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045111334002006023, + "loss": 1.4121, + "theoretical_loss": 4.069771697638838, + "tokens_seen": 352452608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510932798395186, + "loss": 1.5508, + "theoretical_loss": 4.069598795494315, + "tokens_seen": 352583680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045107321965897696, + "loss": 1.3157, + "theoretical_loss": 4.069425975603364, + "tokens_seen": 352714752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510531594784353, + "loss": 1.4493, + "theoretical_loss": 4.0692532378963095, + "tokens_seen": 352845824 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045103309929789374, + "loss": 1.4906, + "theoretical_loss": 4.069080582303563, + "tokens_seen": 352976896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510130391173521, + "loss": 1.5558, + "theoretical_loss": 4.068908008755619, + "tokens_seen": 353107968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045099297893681046, + "loss": 1.4319, + "theoretical_loss": 4.0687355171830575, + "tokens_seen": 353239040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509729187562688, + "loss": 1.6799, + "theoretical_loss": 4.068563107516544, + "tokens_seen": 353370112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509528585757272, + "loss": 1.5795, + "theoretical_loss": 4.068390779686826, + "tokens_seen": 353501184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509327983951856, + "loss": 1.2278, + "theoretical_loss": 4.068218533624737, + "tokens_seen": 353632256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045091273821464396, + "loss": 1.5487, + "theoretical_loss": 4.068046369261194, + "tokens_seen": 353763328 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3893572688102722, + "objective/train/avg_token_score": 0.027582136914134026, + "objective/train/docs_used": 212189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.257407784461975, + "objective/train/original_loss": 3.744044065475464, + "objective/train/theoretical_loss": 4.067874286527197, + "objective/train/tokens_used": 353894400, + "objective/train/unlikelihood_loss": 0.16484889388084412, + "theoretical_loss": 4.067874286527197, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508926780341023, + "loss": 1.5564, + "theoretical_loss": 4.067874286527197, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508726178535607, + "loss": 1.4902, + "theoretical_loss": 4.067702285353833, + "tokens_seen": 354025472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508525576730191, + "loss": 1.4888, + "theoretical_loss": 4.067530365672267, + "tokens_seen": 354156544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045083249749247747, + "loss": 1.4045, + "theoretical_loss": 4.067358527413753, + "tokens_seen": 354287616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045081243731193583, + "loss": 1.5815, + "theoretical_loss": 4.067186770509627, + "tokens_seen": 354418688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507923771313942, + "loss": 1.4217, + "theoretical_loss": 4.067015094891306, + "tokens_seen": 354549760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045077231695085255, + "loss": 1.5362, + "theoretical_loss": 4.066843500490292, + "tokens_seen": 354680832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045075225677031097, + "loss": 1.4491, + "theoretical_loss": 4.066671987238171, + "tokens_seen": 354811904 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045073219658976933, + "loss": 1.3298, + "theoretical_loss": 4.066500555066609, + "tokens_seen": 354942976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507121364092277, + "loss": 1.4452, + "theoretical_loss": 4.066329203907358, + "tokens_seen": 355074048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045069207622868606, + "loss": 1.6151, + "theoretical_loss": 4.066157933692249, + "tokens_seen": 355205120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506720160481445, + "loss": 1.4978, + "theoretical_loss": 4.0659867443532, + "tokens_seen": 355336192 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045065195586760284, + "loss": 1.6458, + "theoretical_loss": 4.065815635822206, + "tokens_seen": 355467264 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5819892287254333, + "objective/train/avg_token_score": 0.017979789525270462, + "objective/train/docs_used": 213521, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.2102198600769043, + "objective/train/original_loss": 4.2459540367126465, + "objective/train/theoretical_loss": 4.065730111838502, + "objective/train/tokens_used": 355532800, + "objective/train/unlikelihood_loss": 0.09294069558382034, + "theoretical_loss": 4.065730111838502, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506318956870612, + "loss": 1.6467, + "theoretical_loss": 4.0656446080313495, + "tokens_seen": 355598336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045061183550651956, + "loss": 1.509, + "theoretical_loss": 4.065473660912791, + "tokens_seen": 355729408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505917753259779, + "loss": 1.3671, + "theoretical_loss": 4.065302794398775, + "tokens_seen": 355860480 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045057171514543634, + "loss": 1.3185, + "theoretical_loss": 4.065132008421628, + "tokens_seen": 355991552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505516549648947, + "loss": 1.4506, + "theoretical_loss": 4.0649613029137575, + "tokens_seen": 356122624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045053159478435306, + "loss": 1.31, + "theoretical_loss": 4.064790677807652, + "tokens_seen": 356253696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505115346038114, + "loss": 1.3887, + "theoretical_loss": 4.064620133035883, + "tokens_seen": 356384768 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045049147442326984, + "loss": 1.4987, + "theoretical_loss": 4.064449668531103, + "tokens_seen": 356515840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504714142427282, + "loss": 1.6307, + "theoretical_loss": 4.064279284226044, + "tokens_seen": 356646912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045045135406218657, + "loss": 1.4173, + "theoretical_loss": 4.06410898005352, + "tokens_seen": 356777984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045043129388164493, + "loss": 1.7445, + "theoretical_loss": 4.0639387559464275, + "tokens_seen": 356909056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045041123370110334, + "loss": 1.7331, + "theoretical_loss": 4.063768611837742, + "tokens_seen": 357040128 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.2592253088951111, + "objective/train/avg_token_score": 0.012151106260716915, + "objective/train/docs_used": 214188, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0094984769821167, + "objective/train/original_loss": 4.309907913208008, + "objective/train/theoretical_loss": 4.063598547660519, + "objective/train/tokens_used": 357171200, + "objective/train/unlikelihood_loss": 0.2334572970867157, + "theoretical_loss": 4.063598547660519, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503911735205617, + "loss": 1.4034, + "theoretical_loss": 4.063598547660519, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045037111334002007, + "loss": 1.5863, + "theoretical_loss": 4.063428563347898, + "tokens_seen": 357302272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045035105315947843, + "loss": 1.4359, + "theoretical_loss": 4.063258658833096, + "tokens_seen": 357433344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503309929789368, + "loss": 1.468, + "theoretical_loss": 4.0630888340494105, + "tokens_seen": 357564416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503109327983952, + "loss": 1.4797, + "theoretical_loss": 4.06291908893022, + "tokens_seen": 357695488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045029087261785357, + "loss": 1.5528, + "theoretical_loss": 4.062749423408981, + "tokens_seen": 357826560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045027081243731193, + "loss": 1.4224, + "theoretical_loss": 4.062579837419235, + "tokens_seen": 357957632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502507522567703, + "loss": 1.5149, + "theoretical_loss": 4.062410330894597, + "tokens_seen": 358088704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502306920762287, + "loss": 1.5902, + "theoretical_loss": 4.062240903768767, + "tokens_seen": 358219776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502106318956871, + "loss": 1.5077, + "theoretical_loss": 4.062071555975519, + "tokens_seen": 358350848 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045019057171514544, + "loss": 1.3566, + "theoretical_loss": 4.061902287448714, + "tokens_seen": 358481920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501705115346038, + "loss": 1.5275, + "theoretical_loss": 4.061733098122284, + "tokens_seen": 358612992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045015045135406216, + "loss": 1.525, + "theoretical_loss": 4.0615639879302465, + "tokens_seen": 358744064 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5088587403297424, + "objective/train/avg_token_score": 0.01710294932126999, + "objective/train/docs_used": 215284, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8704804182052612, + "objective/train/original_loss": 3.966881513595581, + "objective/train/theoretical_loss": 4.061479462489026, + "objective/train/tokens_used": 358809600, + "objective/train/unlikelihood_loss": 0.14359432458877563, + "theoretical_loss": 4.061479462489026, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501303911735206, + "loss": 1.5061, + "theoretical_loss": 4.061394956806694, + "tokens_seen": 358875136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045011033099297894, + "loss": 1.3324, + "theoretical_loss": 4.0612260046858015, + "tokens_seen": 359006208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500902708124373, + "loss": 1.3371, + "theoretical_loss": 4.06105713150182, + "tokens_seen": 359137280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045007021063189567, + "loss": 1.4478, + "theoretical_loss": 4.0608883371890805, + "tokens_seen": 359268352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500501504513541, + "loss": 1.4508, + "theoretical_loss": 4.06071962168199, + "tokens_seen": 359399424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045003009027081244, + "loss": 1.5374, + "theoretical_loss": 4.060550984915039, + "tokens_seen": 359530496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500100300902708, + "loss": 1.3958, + "theoretical_loss": 4.060382426822792, + "tokens_seen": 359661568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044998996990972917, + "loss": 1.4925, + "theoretical_loss": 4.060213947339894, + "tokens_seen": 359792640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044996990972918753, + "loss": 1.4814, + "theoretical_loss": 4.060045546401065, + "tokens_seen": 359923712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044994984954864595, + "loss": 1.5105, + "theoretical_loss": 4.0598772239411085, + "tokens_seen": 360054784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499297893681043, + "loss": 1.3412, + "theoretical_loss": 4.0597089798949, + "tokens_seen": 360185856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044990972918756267, + "loss": 1.443, + "theoretical_loss": 4.059540814197396, + "tokens_seen": 360316928 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3905791640281677, + "objective/train/avg_token_score": 0.048172734677791595, + "objective/train/docs_used": 215704, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4849482774734497, + "objective/train/original_loss": 4.436104774475098, + "objective/train/theoretical_loss": 4.05937272678363, + "objective/train/tokens_used": 360448000, + "objective/train/unlikelihood_loss": 0.11548055708408356, + "theoretical_loss": 4.05937272678363, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044988966900702103, + "loss": 1.4311, + "theoretical_loss": 4.05937272678363, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044986960882647945, + "loss": 1.552, + "theoretical_loss": 4.059204717588712, + "tokens_seen": 360579072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498495486459378, + "loss": 1.4372, + "theoretical_loss": 4.059036786547831, + "tokens_seen": 360710144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498294884653962, + "loss": 1.481, + "theoretical_loss": 4.058868933596252, + "tokens_seen": 360841216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044980942828485454, + "loss": 1.598, + "theoretical_loss": 4.058701158669318, + "tokens_seen": 360972288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497893681043129, + "loss": 1.4096, + "theoretical_loss": 4.0585334617024476, + "tokens_seen": 361103360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497693079237713, + "loss": 1.5581, + "theoretical_loss": 4.058365842631137, + "tokens_seen": 361234432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497492477432297, + "loss": 1.4585, + "theoretical_loss": 4.058198301390959, + "tokens_seen": 361365504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044972918756268804, + "loss": 1.4782, + "theoretical_loss": 4.058030837917565, + "tokens_seen": 361496576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497091273821464, + "loss": 1.2856, + "theoretical_loss": 4.057863452146679, + "tokens_seen": 361627648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496890672016049, + "loss": 1.3921, + "theoretical_loss": 4.057696144014106, + "tokens_seen": 361758720 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044966900702106324, + "loss": 1.6007, + "theoretical_loss": 4.057528913455723, + "tokens_seen": 361889792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496489468405216, + "loss": 1.6407, + "theoretical_loss": 4.0573617604074865, + "tokens_seen": 362020864 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.47562316060066223, + "objective/train/avg_token_score": 0.012415323406457901, + "objective/train/docs_used": 216313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8667166233062744, + "objective/train/original_loss": 4.416706085205078, + "objective/train/theoretical_loss": 4.0572782129296785, + "objective/train/tokens_used": 362086400, + "objective/train/unlikelihood_loss": 0.12603291869163513, + "theoretical_loss": 4.0572782129296785, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044962888665997996, + "loss": 1.5666, + "theoretical_loss": 4.057194684805426, + "tokens_seen": 362151936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496088264794383, + "loss": 1.4589, + "theoretical_loss": 4.0570276865856485, + "tokens_seen": 362283008 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044958876629889674, + "loss": 1.6308, + "theoretical_loss": 4.056860765684338, + "tokens_seen": 362414080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495687061183551, + "loss": 1.5062, + "theoretical_loss": 4.056693922037753, + "tokens_seen": 362545152 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044954864593781346, + "loss": 1.3438, + "theoretical_loss": 4.056527155582226, + "tokens_seen": 362676224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495285857572718, + "loss": 1.3656, + "theoretical_loss": 4.056360466254168, + "tokens_seen": 362807296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044950852557673024, + "loss": 1.5156, + "theoretical_loss": 4.056193853990063, + "tokens_seen": 362938368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494884653961886, + "loss": 1.5408, + "theoretical_loss": 4.056027318726472, + "tokens_seen": 363069440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044946840521564697, + "loss": 1.6677, + "theoretical_loss": 4.0558608604000295, + "tokens_seen": 363200512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044944834503510533, + "loss": 1.4249, + "theoretical_loss": 4.055694478947445, + "tokens_seen": 363331584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044942828485456375, + "loss": 1.4203, + "theoretical_loss": 4.055528174305505, + "tokens_seen": 363462656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494082246740221, + "loss": 1.5913, + "theoretical_loss": 4.055361946411068, + "tokens_seen": 363593728 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.37341153621673584, + "objective/train/avg_token_score": 0.040493469685316086, + "objective/train/docs_used": 217479, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2378941774368286, + "objective/train/original_loss": 3.728134870529175, + "objective/train/theoretical_loss": 4.055195795201069, + "objective/train/tokens_used": 363724800, + "objective/train/unlikelihood_loss": 0.2129121869802475, + "theoretical_loss": 4.055195795201069, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044938816449348047, + "loss": 1.6193, + "theoretical_loss": 4.055195795201069, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044936810431293883, + "loss": 1.465, + "theoretical_loss": 4.0550297206125165, + "tokens_seen": 363855872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493480441323972, + "loss": 1.3898, + "theoretical_loss": 4.054863722582494, + "tokens_seen": 363986944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493279839518556, + "loss": 1.3853, + "theoretical_loss": 4.05469780104816, + "tokens_seen": 364118016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044930792377131397, + "loss": 1.5886, + "theoretical_loss": 4.054531955946745, + "tokens_seen": 364249088 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044928786359077234, + "loss": 1.6761, + "theoretical_loss": 4.054366187215556, + "tokens_seen": 364380160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492678034102307, + "loss": 1.5347, + "theoretical_loss": 4.054200494791972, + "tokens_seen": 364511232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492477432296891, + "loss": 1.3878, + "theoretical_loss": 4.054034878613447, + "tokens_seen": 364642304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492276830491475, + "loss": 1.4606, + "theoretical_loss": 4.053869338617508, + "tokens_seen": 364773376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044920762286860584, + "loss": 1.5716, + "theoretical_loss": 4.053703874741757, + "tokens_seen": 364904448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491875626880642, + "loss": 1.5397, + "theoretical_loss": 4.053538486923868, + "tokens_seen": 365035520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044916750250752256, + "loss": 1.4733, + "theoretical_loss": 4.053373175101589, + "tokens_seen": 365166592 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449147442326981, + "loss": 1.5254, + "theoretical_loss": 4.05320793921274, + "tokens_seen": 365297664 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4299853444099426, + "objective/train/avg_token_score": 0.02031981386244297, + "objective/train/docs_used": 218231, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5544748306274414, + "objective/train/original_loss": 4.155126571655273, + "objective/train/theoretical_loss": 4.053125349723942, + "objective/train/tokens_used": 365363200, + "objective/train/unlikelihood_loss": 0.17956046760082245, + "theoretical_loss": 4.053125349723942, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044912738214643934, + "loss": 1.4319, + "theoretical_loss": 4.053042779195217, + "tokens_seen": 365428736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491073219658977, + "loss": 1.4881, + "theoretical_loss": 4.052877694986987, + "tokens_seen": 365559808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044908726178535607, + "loss": 1.4094, + "theoretical_loss": 4.05271268652609, + "tokens_seen": 365690880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490672016048145, + "loss": 1.671, + "theoretical_loss": 4.052547753750639, + "tokens_seen": 365821952 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044904714142427284, + "loss": 1.4686, + "theoretical_loss": 4.05238289659882, + "tokens_seen": 365953024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490270812437312, + "loss": 1.3926, + "theoretical_loss": 4.052218115008892, + "tokens_seen": 366084096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044900702106318957, + "loss": 1.5302, + "theoretical_loss": 4.0520534089191855, + "tokens_seen": 366215168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044898696088264793, + "loss": 1.6902, + "theoretical_loss": 4.051888778268103, + "tokens_seen": 366346240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044896690070210635, + "loss": 1.5773, + "theoretical_loss": 4.051724222994122, + "tokens_seen": 366477312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489468405215647, + "loss": 1.4207, + "theoretical_loss": 4.051559743035789, + "tokens_seen": 366608384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044892678034102307, + "loss": 1.513, + "theoretical_loss": 4.051395338331723, + "tokens_seen": 366739456 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044890672016048143, + "loss": 1.4494, + "theoretical_loss": 4.051231008820618, + "tokens_seen": 366870528 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.34115347266197205, + "objective/train/avg_token_score": 0.10597312450408936, + "objective/train/docs_used": 219467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.169438123703003, + "objective/train/original_loss": 5.253897190093994, + "objective/train/theoretical_loss": 4.051066754441235, + "objective/train/tokens_used": 367001600, + "objective/train/unlikelihood_loss": 0.0806141048669815, + "theoretical_loss": 4.051066754441235, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044888665997993985, + "loss": 1.3879, + "theoretical_loss": 4.051066754441235, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488665997993982, + "loss": 1.6292, + "theoretical_loss": 4.050902575132412, + "tokens_seen": 367132672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488465396188566, + "loss": 1.5422, + "theoretical_loss": 4.050738470833054, + "tokens_seen": 367263744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044882647943831494, + "loss": 1.5571, + "theoretical_loss": 4.05057444148214, + "tokens_seen": 367394816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488064192577733, + "loss": 1.5531, + "theoretical_loss": 4.05041048701872, + "tokens_seen": 367525888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487863590772317, + "loss": 1.4186, + "theoretical_loss": 4.050246607381913, + "tokens_seen": 367656960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487662988966901, + "loss": 1.6269, + "theoretical_loss": 4.050082802510915, + "tokens_seen": 367788032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044874623871614844, + "loss": 1.5642, + "theoretical_loss": 4.049919072344987, + "tokens_seen": 367919104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487261785356068, + "loss": 1.5353, + "theoretical_loss": 4.049755416823463, + "tokens_seen": 368050176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487061183550652, + "loss": 1.4776, + "theoretical_loss": 4.0495918358857494, + "tokens_seen": 368181248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486860581745236, + "loss": 1.465, + "theoretical_loss": 4.04942832947132, + "tokens_seen": 368312320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044866599799398194, + "loss": 1.5645, + "theoretical_loss": 4.049264897519723, + "tokens_seen": 368443392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486459378134403, + "loss": 1.425, + "theoretical_loss": 4.049101539970576, + "tokens_seen": 368574464 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.47611191868782043, + "objective/train/avg_token_score": 0.012634911574423313, + "objective/train/docs_used": 220098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.645254135131836, + "objective/train/original_loss": 4.079015731811523, + "objective/train/theoretical_loss": 4.04901988907807, + "objective/train/tokens_used": 368640000, + "objective/train/unlikelihood_loss": 0.1734001785516739, + "theoretical_loss": 4.04901988907807, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044862587763289867, + "loss": 1.7137, + "theoretical_loss": 4.048938256763565, + "tokens_seen": 368705536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486058174523571, + "loss": 1.6932, + "theoretical_loss": 4.048775047838447, + "tokens_seen": 368836608 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044858575727181545, + "loss": 1.705, + "theoretical_loss": 4.048611913135052, + "tokens_seen": 368967680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485656970912738, + "loss": 1.7297, + "theoretical_loss": 4.048448852593277, + "tokens_seen": 369098752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044854563691073217, + "loss": 1.7156, + "theoretical_loss": 4.0482858661530905, + "tokens_seen": 369229824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485255767301906, + "loss": 1.4436, + "theoretical_loss": 4.04812295375453, + "tokens_seen": 369360896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044850551654964895, + "loss": 1.5983, + "theoretical_loss": 4.047960115337702, + "tokens_seen": 369491968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484854563691073, + "loss": 1.5025, + "theoretical_loss": 4.047797350842786, + "tokens_seen": 369623040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484653961885657, + "loss": 1.5012, + "theoretical_loss": 4.0476346602100275, + "tokens_seen": 369754112 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044844533600802404, + "loss": 1.5321, + "theoretical_loss": 4.047472043379743, + "tokens_seen": 369885184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044842527582748245, + "loss": 1.3779, + "theoretical_loss": 4.04730950029232, + "tokens_seen": 370016256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484052156469408, + "loss": 1.7209, + "theoretical_loss": 4.04714703088821, + "tokens_seen": 370147328 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4204545319080353, + "objective/train/avg_token_score": 0.013336265459656715, + "objective/train/docs_used": 221122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5654873847961426, + "objective/train/original_loss": 4.028966903686523, + "objective/train/theoretical_loss": 4.04698463510794, + "objective/train/tokens_used": 370278400, + "objective/train/unlikelihood_loss": 0.19616243243217468, + "theoretical_loss": 4.04698463510794, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483851554663992, + "loss": 1.3742, + "theoretical_loss": 4.04698463510794, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044836509528585754, + "loss": 1.3853, + "theoretical_loss": 4.046822312892102, + "tokens_seen": 370409472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044834503510531596, + "loss": 1.5133, + "theoretical_loss": 4.046660064181359, + "tokens_seen": 370540544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483249749247743, + "loss": 1.4851, + "theoretical_loss": 4.0464978889164405, + "tokens_seen": 370671616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483049147442327, + "loss": 1.5003, + "theoretical_loss": 4.046335787038148, + "tokens_seen": 370802688 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828485456369104, + "loss": 1.6905, + "theoretical_loss": 4.046173758487347, + "tokens_seen": 370933760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482647943831494, + "loss": 1.4056, + "theoretical_loss": 4.046011803204976, + "tokens_seen": 371064832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482447342026078, + "loss": 1.4829, + "theoretical_loss": 4.045849921132042, + "tokens_seen": 371195904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482246740220662, + "loss": 1.6773, + "theoretical_loss": 4.045688112209615, + "tokens_seen": 371326976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044820461384152455, + "loss": 1.4661, + "theoretical_loss": 4.04552637637884, + "tokens_seen": 371458048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044818455366098296, + "loss": 1.6595, + "theoretical_loss": 4.045364713580923, + "tokens_seen": 371589120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481644934804414, + "loss": 1.6206, + "theoretical_loss": 4.045203123757146, + "tokens_seen": 371720192 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044814443329989974, + "loss": 1.6652, + "theoretical_loss": 4.0450416068488515, + "tokens_seen": 371851264 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4794110357761383, + "objective/train/avg_token_score": 0.004854011349380016, + "objective/train/docs_used": 221664, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7908248901367188, + "objective/train/original_loss": 3.7173187732696533, + "objective/train/theoretical_loss": 4.044960875719701, + "objective/train/tokens_used": 371916800, + "objective/train/unlikelihood_loss": 0.17050793766975403, + "theoretical_loss": 4.044960875719701, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481243731193581, + "loss": 1.493, + "theoretical_loss": 4.0448801627974555, + "tokens_seen": 371982336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044810431293881647, + "loss": 1.5649, + "theoretical_loss": 4.044718791544438, + "tokens_seen": 372113408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480842527582749, + "loss": 1.7985, + "theoretical_loss": 4.044557493031347, + "tokens_seen": 372244480 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044806419257773324, + "loss": 1.5757, + "theoretical_loss": 4.0443962671998, + "tokens_seen": 372375552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480441323971916, + "loss": 1.5035, + "theoretical_loss": 4.04423511399148, + "tokens_seen": 372506624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044802407221664997, + "loss": 1.5995, + "theoretical_loss": 4.044074033348139, + "tokens_seen": 372637696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044800401203610833, + "loss": 1.7926, + "theoretical_loss": 4.0439130252115945, + "tokens_seen": 372768768 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044798395185556675, + "loss": 1.7443, + "theoretical_loss": 4.043752089523732, + "tokens_seen": 372899840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479638916750251, + "loss": 1.6664, + "theoretical_loss": 4.043591226226503, + "tokens_seen": 373030912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044794383149448347, + "loss": 1.55, + "theoretical_loss": 4.043430435261927, + "tokens_seen": 373161984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044792377131394183, + "loss": 1.5638, + "theoretical_loss": 4.043269716572091, + "tokens_seen": 373293056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044790371113340025, + "loss": 1.5155, + "theoretical_loss": 4.043109070099146, + "tokens_seen": 373424128 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5913978219032288, + "objective/train/avg_token_score": 0.00555779505521059, + "objective/train/docs_used": 222906, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1588377952575684, + "objective/train/original_loss": 3.770258665084839, + "objective/train/theoretical_loss": 4.042948495785312, + "objective/train/tokens_used": 373555200, + "objective/train/unlikelihood_loss": 0.16115084290504456, + "theoretical_loss": 4.042948495785312, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478836509528586, + "loss": 1.5525, + "theoretical_loss": 4.042948495785312, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447863590772317, + "loss": 1.5479, + "theoretical_loss": 4.042787993572875, + "tokens_seen": 373686272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044784353059177534, + "loss": 1.4116, + "theoretical_loss": 4.042627563404187, + "tokens_seen": 373817344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478234704112337, + "loss": 1.5096, + "theoretical_loss": 4.042467205221665, + "tokens_seen": 373948416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478034102306921, + "loss": 1.5331, + "theoretical_loss": 4.042306918967795, + "tokens_seen": 374079488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477833500501505, + "loss": 1.6504, + "theoretical_loss": 4.042146704585127, + "tokens_seen": 374210560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044776328986960884, + "loss": 1.6323, + "theoretical_loss": 4.041986562016278, + "tokens_seen": 374341632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477432296890672, + "loss": 1.4255, + "theoretical_loss": 4.0418264912039295, + "tokens_seen": 374472704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477231695085256, + "loss": 1.7105, + "theoretical_loss": 4.041666492090831, + "tokens_seen": 374603776 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447703109327984, + "loss": 1.4864, + "theoretical_loss": 4.041506564619796, + "tokens_seen": 374734848 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044768304914744234, + "loss": 1.5879, + "theoretical_loss": 4.041346708733704, + "tokens_seen": 374865920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476629889669007, + "loss": 1.6726, + "theoretical_loss": 4.041186924375499, + "tokens_seen": 374996992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044764292878635907, + "loss": 1.7443, + "theoretical_loss": 4.041027211488194, + "tokens_seen": 375128064 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.2611192464828491, + "objective/train/avg_token_score": 0.02251238189637661, + "objective/train/docs_used": 223164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0515928268432617, + "objective/train/original_loss": 4.479992389678955, + "objective/train/theoretical_loss": 4.040947381828337, + "objective/train/tokens_used": 375193600, + "objective/train/unlikelihood_loss": 0.1949537694454193, + "theoretical_loss": 4.040947381828337, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476228686058175, + "loss": 1.406, + "theoretical_loss": 4.040867570014862, + "tokens_seen": 375259136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044760280842527585, + "loss": 1.4454, + "theoretical_loss": 4.040707999898647, + "tokens_seen": 375390208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475827482447342, + "loss": 1.7935, + "theoretical_loss": 4.0405485010827515, + "tokens_seen": 375521280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044756268806419257, + "loss": 1.5507, + "theoretical_loss": 4.040389073510449, + "tokens_seen": 375652352 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447542627883651, + "loss": 1.3859, + "theoretical_loss": 4.040229717125074, + "tokens_seen": 375783424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044752256770310935, + "loss": 1.6392, + "theoretical_loss": 4.040070431870029, + "tokens_seen": 375914496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475025075225677, + "loss": 1.5432, + "theoretical_loss": 4.039911217688777, + "tokens_seen": 376045568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474824473420261, + "loss": 1.4319, + "theoretical_loss": 4.039752074524851, + "tokens_seen": 376176640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044746238716148444, + "loss": 1.5912, + "theoretical_loss": 4.039593002321844, + "tokens_seen": 376307712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044744232698094285, + "loss": 1.5599, + "theoretical_loss": 4.039434001023415, + "tokens_seen": 376438784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474222668004012, + "loss": 1.3618, + "theoretical_loss": 4.039275070573288, + "tokens_seen": 376569856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474022066198596, + "loss": 1.5952, + "theoretical_loss": 4.03911621091525, + "tokens_seen": 376700928 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4695136845111847, + "objective/train/avg_token_score": 0.009801458567380905, + "objective/train/docs_used": 224842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.732696771621704, + "objective/train/original_loss": 4.10999059677124, + "objective/train/theoretical_loss": 4.038957421993153, + "objective/train/tokens_used": 376832000, + "objective/train/unlikelihood_loss": 0.14230914413928986, + "theoretical_loss": 4.038957421993153, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044738214643931794, + "loss": 1.7543, + "theoretical_loss": 4.038957421993153, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044736208625877636, + "loss": 1.4096, + "theoretical_loss": 4.038798703750913, + "tokens_seen": 376963072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473420260782347, + "loss": 1.6097, + "theoretical_loss": 4.038640056132509, + "tokens_seen": 377094144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473219658976931, + "loss": 1.6665, + "theoretical_loss": 4.0384814790819865, + "tokens_seen": 377225216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044730190571715144, + "loss": 1.5166, + "theoretical_loss": 4.038322972543451, + "tokens_seen": 377356288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472818455366098, + "loss": 1.7482, + "theoretical_loss": 4.038164536461075, + "tokens_seen": 377487360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472617853560682, + "loss": 1.5796, + "theoretical_loss": 4.038006170779091, + "tokens_seen": 377618432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472417251755266, + "loss": 1.4874, + "theoretical_loss": 4.037847875441798, + "tokens_seen": 377749504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044722166499498495, + "loss": 1.6282, + "theoretical_loss": 4.037689650393559, + "tokens_seen": 377880576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472016048144433, + "loss": 1.6695, + "theoretical_loss": 4.037531495578797, + "tokens_seen": 378011648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471815446339017, + "loss": 1.4349, + "theoretical_loss": 4.037373410942001, + "tokens_seen": 378142720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471614844533601, + "loss": 1.7403, + "theoretical_loss": 4.037215396427721, + "tokens_seen": 378273792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044714142427281845, + "loss": 1.5023, + "theoretical_loss": 4.037057451980571, + "tokens_seen": 378404864 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4757453501224518, + "objective/train/avg_token_score": 0.021771633997559547, + "objective/train/docs_used": 225570, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.804304599761963, + "objective/train/original_loss": 4.102111339569092, + "objective/train/theoretical_loss": 4.036978506014879, + "objective/train/tokens_used": 378470400, + "objective/train/unlikelihood_loss": 0.18979206681251526, + "theoretical_loss": 4.036978506014879, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471213640922768, + "loss": 1.7116, + "theoretical_loss": 4.036899577545229, + "tokens_seen": 378535936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044710130391173523, + "loss": 1.6378, + "theoretical_loss": 4.036741773066434, + "tokens_seen": 378667008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470812437311936, + "loss": 1.6524, + "theoretical_loss": 4.036584038488988, + "tokens_seen": 378798080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044706118355065195, + "loss": 1.4253, + "theoretical_loss": 4.036426373757759, + "tokens_seen": 378929152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470411233701103, + "loss": 1.6715, + "theoretical_loss": 4.0362687788176705, + "tokens_seen": 379060224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470210631895687, + "loss": 1.8991, + "theoretical_loss": 4.036111253613715, + "tokens_seen": 379191296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470010030090271, + "loss": 1.6512, + "theoretical_loss": 4.0359537980909455, + "tokens_seen": 379322368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044698094282848546, + "loss": 1.4178, + "theoretical_loss": 4.0357964121944745, + "tokens_seen": 379453440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469608826479438, + "loss": 1.5667, + "theoretical_loss": 4.035639095869481, + "tokens_seen": 379584512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469408224674022, + "loss": 1.5906, + "theoretical_loss": 4.0354818490612026, + "tokens_seen": 379715584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469207622868606, + "loss": 1.5448, + "theoretical_loss": 4.035324671714942, + "tokens_seen": 379846656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044690070210631896, + "loss": 1.6204, + "theoretical_loss": 4.03516756377606, + "tokens_seen": 379977728 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4055474102497101, + "objective/train/avg_token_score": 0.01529501099139452, + "objective/train/docs_used": 226798, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4889705181121826, + "objective/train/original_loss": 3.754312515258789, + "objective/train/theoretical_loss": 4.035010525189982, + "objective/train/tokens_used": 380108800, + "objective/train/unlikelihood_loss": 0.2511163651943207, + "theoretical_loss": 4.035010525189982, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468806419257773, + "loss": 1.5714, + "theoretical_loss": 4.035010525189982, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468605817452357, + "loss": 1.3369, + "theoretical_loss": 4.0348535559021945, + "tokens_seen": 380239872 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044684052156469405, + "loss": 1.5174, + "theoretical_loss": 4.034696655858246, + "tokens_seen": 380370944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044682046138415246, + "loss": 1.607, + "theoretical_loss": 4.034539825003746, + "tokens_seen": 380502016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468004012036108, + "loss": 1.5884, + "theoretical_loss": 4.034383063284364, + "tokens_seen": 380633088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467803410230692, + "loss": 1.5842, + "theoretical_loss": 4.0342263706458334, + "tokens_seen": 380764160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044676028084252755, + "loss": 1.6033, + "theoretical_loss": 4.034069747033948, + "tokens_seen": 380895232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044674022066198597, + "loss": 1.5395, + "theoretical_loss": 4.033913192394563, + "tokens_seen": 381026304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044672016048144433, + "loss": 1.7102, + "theoretical_loss": 4.033756706673593, + "tokens_seen": 381157376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467001003009027, + "loss": 1.5634, + "theoretical_loss": 4.033600289817015, + "tokens_seen": 381288448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466800401203611, + "loss": 1.4824, + "theoretical_loss": 4.033443941770867, + "tokens_seen": 381419520 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044665997993981947, + "loss": 1.6288, + "theoretical_loss": 4.033287662481247, + "tokens_seen": 381550592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466399197592779, + "loss": 1.653, + "theoretical_loss": 4.033131451894317, + "tokens_seen": 381681664 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4856427311897278, + "objective/train/avg_token_score": 0.012365730479359627, + "objective/train/docs_used": 227456, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6744704246520996, + "objective/train/original_loss": 3.758188486099243, + "objective/train/theoretical_loss": 4.03305337234755, + "objective/train/tokens_used": 381747200, + "objective/train/unlikelihood_loss": 0.1711551696062088, + "theoretical_loss": 4.03305337234755, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044661985957873625, + "loss": 1.5343, + "theoretical_loss": 4.032975309956292, + "tokens_seen": 381812736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465997993981946, + "loss": 1.5904, + "theoretical_loss": 4.032819236613457, + "tokens_seen": 381943808 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044657973921765297, + "loss": 1.4637, + "theoretical_loss": 4.03266323181215, + "tokens_seen": 382074880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465596790371114, + "loss": 1.5051, + "theoretical_loss": 4.032507295498774, + "tokens_seen": 382205952 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044653961885656975, + "loss": 1.6082, + "theoretical_loss": 4.032351427619789, + "tokens_seen": 382337024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465195586760281, + "loss": 1.5642, + "theoretical_loss": 4.032195628121718, + "tokens_seen": 382468096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464994984954865, + "loss": 1.5348, + "theoretical_loss": 4.032039896951142, + "tokens_seen": 382599168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044647943831494484, + "loss": 1.6252, + "theoretical_loss": 4.0318842340547025, + "tokens_seen": 382730240 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044645937813440325, + "loss": 1.4619, + "theoretical_loss": 4.031728639379103, + "tokens_seen": 382861312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464393179538616, + "loss": 1.4193, + "theoretical_loss": 4.031573112871103, + "tokens_seen": 382992384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044641925777332, + "loss": 1.7358, + "theoretical_loss": 4.031417654477526, + "tokens_seen": 383123456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044639919759277834, + "loss": 1.6106, + "theoretical_loss": 4.03126226414525, + "tokens_seen": 383254528 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4290078282356262, + "objective/train/avg_token_score": 0.0078100161626935005, + "objective/train/docs_used": 228876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6035910844802856, + "objective/train/original_loss": 4.235310077667236, + "objective/train/theoretical_loss": 4.031106941821218, + "objective/train/tokens_used": 383385600, + "objective/train/unlikelihood_loss": 0.13287082314491272, + "theoretical_loss": 4.031106941821218, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044637913741223676, + "loss": 1.7171, + "theoretical_loss": 4.031106941821218, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463590772316951, + "loss": 1.6425, + "theoretical_loss": 4.030951687452429, + "tokens_seen": 383516672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463390170511535, + "loss": 1.5361, + "theoretical_loss": 4.030796500985943, + "tokens_seen": 383647744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044631895687061184, + "loss": 1.6445, + "theoretical_loss": 4.030641382368878, + "tokens_seen": 383778816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462988966900702, + "loss": 1.5612, + "theoretical_loss": 4.030486331548413, + "tokens_seen": 383909888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462788365095286, + "loss": 1.649, + "theoretical_loss": 4.030331348471784, + "tokens_seen": 384040960 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446258776328987, + "loss": 1.4784, + "theoretical_loss": 4.030176433086289, + "tokens_seen": 384172032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044623871614844535, + "loss": 1.4671, + "theoretical_loss": 4.030021585339281, + "tokens_seen": 384303104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462186559679037, + "loss": 1.7199, + "theoretical_loss": 4.029866805178175, + "tokens_seen": 384434176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461985957873621, + "loss": 1.4296, + "theoretical_loss": 4.029712092550443, + "tokens_seen": 384565248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461785356068205, + "loss": 1.4691, + "theoretical_loss": 4.029557447403618, + "tokens_seen": 384696320 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044615847542627885, + "loss": 1.4171, + "theoretical_loss": 4.029402869685288, + "tokens_seen": 384827392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461384152457372, + "loss": 1.4194, + "theoretical_loss": 4.029248359343105, + "tokens_seen": 384958464 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.589442789554596, + "objective/train/avg_token_score": 0.008675182238221169, + "objective/train/docs_used": 229454, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1350181102752686, + "objective/train/original_loss": 3.9059112071990967, + "objective/train/theoretical_loss": 4.029171129421724, + "objective/train/tokens_used": 385024000, + "objective/train/unlikelihood_loss": 0.1192452609539032, + "theoretical_loss": 4.029171129421724, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044611835506519563, + "loss": 1.8329, + "theoretical_loss": 4.029093916324774, + "tokens_seen": 385089536 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446098294884654, + "loss": 1.5253, + "theoretical_loss": 4.02893954057806, + "tokens_seen": 385220608 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044607823470411235, + "loss": 1.5488, + "theoretical_loss": 4.0287852320507875, + "tokens_seen": 385351680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460581745235707, + "loss": 1.6576, + "theoretical_loss": 4.028630990690838, + "tokens_seen": 385482752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460381143430291, + "loss": 1.5913, + "theoretical_loss": 4.028476816446153, + "tokens_seen": 385613824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460180541624875, + "loss": 1.5977, + "theoretical_loss": 4.028322709264729, + "tokens_seen": 385744896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044599799398194586, + "loss": 1.4783, + "theoretical_loss": 4.028168669094623, + "tokens_seen": 385875968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459779338014042, + "loss": 1.3898, + "theoretical_loss": 4.0280146958839484, + "tokens_seen": 386007040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459578736208626, + "loss": 1.3856, + "theoretical_loss": 4.027860789580878, + "tokens_seen": 386138112 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445937813440321, + "loss": 1.3978, + "theoretical_loss": 4.0277069501336396, + "tokens_seen": 386269184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591775325977936, + "loss": 1.4399, + "theoretical_loss": 4.027553177490521, + "tokens_seen": 386400256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458976930792377, + "loss": 1.5377, + "theoretical_loss": 4.027399471599867, + "tokens_seen": 386531328 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.48521506786346436, + "objective/train/avg_token_score": 0.010920585133135319, + "objective/train/docs_used": 230121, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8535081148147583, + "objective/train/original_loss": 4.084407329559326, + "objective/train/theoretical_loss": 4.027245832410079, + "objective/train/tokens_used": 386662400, + "objective/train/unlikelihood_loss": 0.12329123914241791, + "theoretical_loss": 4.027245832410079, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458776328986961, + "loss": 1.6056, + "theoretical_loss": 4.027245832410079, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044585757271815445, + "loss": 1.4379, + "theoretical_loss": 4.027092259869616, + "tokens_seen": 386793472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044583751253761286, + "loss": 1.6012, + "theoretical_loss": 4.026938753926997, + "tokens_seen": 386924544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458174523570712, + "loss": 1.5904, + "theoretical_loss": 4.026785314530792, + "tokens_seen": 387055616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457973921765296, + "loss": 1.5274, + "theoretical_loss": 4.026631941629635, + "tokens_seen": 387186688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044577733199598795, + "loss": 1.6511, + "theoretical_loss": 4.026478635172212, + "tokens_seen": 387317760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575727181544637, + "loss": 1.4566, + "theoretical_loss": 4.0263253951072695, + "tokens_seen": 387448832 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044573721163490473, + "loss": 1.3738, + "theoretical_loss": 4.026172221383607, + "tokens_seen": 387579904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457171514543631, + "loss": 1.3676, + "theoretical_loss": 4.026019113950086, + "tokens_seen": 387710976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044569709127382145, + "loss": 1.5872, + "theoretical_loss": 4.025866072755619, + "tokens_seen": 387842048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456770310932798, + "loss": 1.509, + "theoretical_loss": 4.02571309774918, + "tokens_seen": 387973120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044565697091273823, + "loss": 1.4532, + "theoretical_loss": 4.025560188879796, + "tokens_seen": 388104192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456369107321966, + "loss": 1.572, + "theoretical_loss": 4.025407346096551, + "tokens_seen": 388235264 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.3402370512485504, + "objective/train/avg_token_score": 0.010277241468429565, + "objective/train/docs_used": 231531, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1777230501174927, + "objective/train/original_loss": 4.140812397003174, + "objective/train/theoretical_loss": 4.025330949471336, + "objective/train/tokens_used": 388300800, + "objective/train/unlikelihood_loss": 0.12250532954931259, + "theoretical_loss": 4.025330949471336, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044561685055165496, + "loss": 1.5369, + "theoretical_loss": 4.025254569348588, + "tokens_seen": 388366336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455967903711133, + "loss": 1.5416, + "theoretical_loss": 4.025101858585104, + "tokens_seen": 388497408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044557673019057173, + "loss": 1.5914, + "theoretical_loss": 4.0249492137553515, + "tokens_seen": 388628480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455566700100301, + "loss": 1.3606, + "theoretical_loss": 4.0247966348086415, + "tokens_seen": 388759552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044553660982948846, + "loss": 1.5075, + "theoretical_loss": 4.02464412169434, + "tokens_seen": 388890624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455165496489468, + "loss": 1.5384, + "theoretical_loss": 4.024491674361868, + "tokens_seen": 389021696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454964894684052, + "loss": 1.4394, + "theoretical_loss": 4.024339292760703, + "tokens_seen": 389152768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454764292878636, + "loss": 1.5151, + "theoretical_loss": 4.0241869768403795, + "tokens_seen": 389283840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044545636910732196, + "loss": 1.6464, + "theoretical_loss": 4.024034726550486, + "tokens_seen": 389414912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454363089267803, + "loss": 1.5365, + "theoretical_loss": 4.023882541840667, + "tokens_seen": 389545984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454162487462387, + "loss": 1.4798, + "theoretical_loss": 4.023730422660624, + "tokens_seen": 389677056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453961885656971, + "loss": 1.4354, + "theoretical_loss": 4.023578368960113, + "tokens_seen": 389808128 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.34365835785865784, + "objective/train/avg_token_score": 0.01555408164858818, + "objective/train/docs_used": 232298, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2046136856079102, + "objective/train/original_loss": 3.8979644775390625, + "objective/train/theoretical_loss": 4.023426380688943, + "objective/train/tokens_used": 389939200, + "objective/train/unlikelihood_loss": 0.1804252713918686, + "theoretical_loss": 4.023426380688943, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044537612838515547, + "loss": 1.3799, + "theoretical_loss": 4.023426380688943, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044535606820461383, + "loss": 1.6791, + "theoretical_loss": 4.023274457796983, + "tokens_seen": 390070272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453360080240722, + "loss": 1.45, + "theoretical_loss": 4.023122600234155, + "tokens_seen": 390201344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531594784353055, + "loss": 1.5732, + "theoretical_loss": 4.022970807950434, + "tokens_seen": 390332416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044529588766298897, + "loss": 1.5985, + "theoretical_loss": 4.022819080895853, + "tokens_seen": 390463488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527582748244733, + "loss": 1.5921, + "theoretical_loss": 4.022667419020499, + "tokens_seen": 390594560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452557673019057, + "loss": 1.5946, + "theoretical_loss": 4.022515822274515, + "tokens_seen": 390725632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044523570712136406, + "loss": 1.5798, + "theoretical_loss": 4.022364290608098, + "tokens_seen": 390856704 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044521564694082247, + "loss": 1.4749, + "theoretical_loss": 4.022212823971498, + "tokens_seen": 390987776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044519558676028083, + "loss": 1.7443, + "theoretical_loss": 4.022061422315023, + "tokens_seen": 391118848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044517552657973925, + "loss": 1.5852, + "theoretical_loss": 4.021910085589032, + "tokens_seen": 391249920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451554663991976, + "loss": 1.6875, + "theoretical_loss": 4.021758813743942, + "tokens_seen": 391380992 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044513540621865603, + "loss": 1.4254, + "theoretical_loss": 4.021607606730223, + "tokens_seen": 391512064 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.3057795763015747, + "objective/train/avg_token_score": 0.016242092475295067, + "objective/train/docs_used": 233436, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1886520385742188, + "objective/train/original_loss": 4.423647880554199, + "objective/train/theoretical_loss": 4.0215320275196635, + "objective/train/tokens_used": 391577600, + "objective/train/unlikelihood_loss": 0.1317605823278427, + "theoretical_loss": 4.0215320275196635, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451153460381144, + "loss": 1.3848, + "theoretical_loss": 4.0214564644983986, + "tokens_seen": 391643136 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044509528585757275, + "loss": 1.7228, + "theoretical_loss": 4.021305386999048, + "tokens_seen": 391774208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450752256770311, + "loss": 1.475, + "theoretical_loss": 4.021154374182803, + "tokens_seen": 391905280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450551654964895, + "loss": 1.4708, + "theoretical_loss": 4.021003426000353, + "tokens_seen": 392036352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450351053159479, + "loss": 1.7547, + "theoretical_loss": 4.0208525424024355, + "tokens_seen": 392167424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044501504513540626, + "loss": 1.5899, + "theoretical_loss": 4.020701723339848, + "tokens_seen": 392298496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449949849548646, + "loss": 1.4629, + "theoretical_loss": 4.020550968763439, + "tokens_seen": 392429568 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444974924774323, + "loss": 1.5781, + "theoretical_loss": 4.020400278624112, + "tokens_seen": 392560640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449548645937814, + "loss": 1.3149, + "theoretical_loss": 4.020249652872822, + "tokens_seen": 392691712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044493480441323976, + "loss": 1.7288, + "theoretical_loss": 4.02009909146058, + "tokens_seen": 392822784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449147442326981, + "loss": 1.6467, + "theoretical_loss": 4.01994859433845, + "tokens_seen": 392953856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448946840521565, + "loss": 1.7551, + "theoretical_loss": 4.019798161457548, + "tokens_seen": 393084928 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4335899353027344, + "objective/train/avg_token_score": 0.016622694209218025, + "objective/train/docs_used": 234056, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6204777956008911, + "objective/train/original_loss": 3.885645866394043, + "objective/train/theoretical_loss": 4.019647792769048, + "objective/train/tokens_used": 393216000, + "objective/train/unlikelihood_loss": 0.16480211913585663, + "theoretical_loss": 4.019647792769048, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044487462387161485, + "loss": 1.5896, + "theoretical_loss": 4.019647792769048, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044485456369107326, + "loss": 1.5453, + "theoretical_loss": 4.01949748822417, + "tokens_seen": 393347072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448345035105316, + "loss": 1.4782, + "theoretical_loss": 4.019347247774195, + "tokens_seen": 393478144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044481444332999, + "loss": 1.4937, + "theoretical_loss": 4.019197071370451, + "tokens_seen": 393609216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044479438314944835, + "loss": 1.438, + "theoretical_loss": 4.019046958964324, + "tokens_seen": 393740288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044477432296890677, + "loss": 1.3938, + "theoretical_loss": 4.01889691050725, + "tokens_seen": 393871360 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044475426278836513, + "loss": 1.7109, + "theoretical_loss": 4.018746925950719, + "tokens_seen": 394002432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447342026078235, + "loss": 1.3604, + "theoretical_loss": 4.018597005246273, + "tokens_seen": 394133504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044471414242728185, + "loss": 1.6068, + "theoretical_loss": 4.01844714834551, + "tokens_seen": 394264576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446940822467402, + "loss": 1.5331, + "theoretical_loss": 4.018297355200076, + "tokens_seen": 394395648 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044467402206619863, + "loss": 1.465, + "theoretical_loss": 4.018147625761673, + "tokens_seen": 394526720 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444653961885657, + "loss": 1.5658, + "theoretical_loss": 4.017997959982055, + "tokens_seen": 394657792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044463390170511536, + "loss": 1.7217, + "theoretical_loss": 4.017848357813028, + "tokens_seen": 394788864 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.45723363757133484, + "objective/train/avg_token_score": 0.017939608544111252, + "objective/train/docs_used": 235431, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7509286403656006, + "objective/train/original_loss": 4.263587474822998, + "objective/train/theoretical_loss": 4.01777358056744, + "objective/train/tokens_used": 394854400, + "objective/train/unlikelihood_loss": 0.13543236255645752, + "theoretical_loss": 4.01777358056744, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446138415245737, + "loss": 1.635, + "theoretical_loss": 4.017698819206451, + "tokens_seen": 394919936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044459378134403214, + "loss": 1.5588, + "theoretical_loss": 4.017549344114236, + "tokens_seen": 395051008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445737211634905, + "loss": 1.5377, + "theoretical_loss": 4.0173999324883445, + "tokens_seen": 395182080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044455366098294886, + "loss": 1.5516, + "theoretical_loss": 4.017250584280794, + "tokens_seen": 395313152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445336008024072, + "loss": 1.5806, + "theoretical_loss": 4.0171012994436515, + "tokens_seen": 395444224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445135406218656, + "loss": 1.574, + "theoretical_loss": 4.016952077929037, + "tokens_seen": 395575296 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444493480441324, + "loss": 1.7532, + "theoretical_loss": 4.016802919689124, + "tokens_seen": 395706368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044447342026078236, + "loss": 1.4473, + "theoretical_loss": 4.016653824676135, + "tokens_seen": 395837440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444533600802407, + "loss": 1.5449, + "theoretical_loss": 4.016504792842346, + "tokens_seen": 395968512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444332998996991, + "loss": 1.7389, + "theoretical_loss": 4.016355824140085, + "tokens_seen": 396099584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444132397191575, + "loss": 1.6257, + "theoretical_loss": 4.016206918521731, + "tokens_seen": 396230656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044439317953861587, + "loss": 1.5358, + "theoretical_loss": 4.016058075939716, + "tokens_seen": 396361728 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.3450024425983429, + "objective/train/avg_token_score": 0.024041682481765747, + "objective/train/docs_used": 235840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.240441083908081, + "objective/train/original_loss": 3.7224650382995605, + "objective/train/theoretical_loss": 4.015909296346521, + "objective/train/tokens_used": 396492800, + "objective/train/unlikelihood_loss": 0.21158063411712646, + "theoretical_loss": 4.015909296346521, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044437311935807423, + "loss": 1.5572, + "theoretical_loss": 4.015909296346521, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443530591775326, + "loss": 1.5529, + "theoretical_loss": 4.015760579694683, + "tokens_seen": 396623872 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044433299899699095, + "loss": 1.5808, + "theoretical_loss": 4.015611925936785, + "tokens_seen": 396754944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044431293881644937, + "loss": 1.449, + "theoretical_loss": 4.015463335025465, + "tokens_seen": 396886016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044429287863590773, + "loss": 1.3055, + "theoretical_loss": 4.015314806913411, + "tokens_seen": 397017088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442728184553661, + "loss": 1.3535, + "theoretical_loss": 4.015166341553362, + "tokens_seen": 397148160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044425275827482446, + "loss": 1.536, + "theoretical_loss": 4.01501793889811, + "tokens_seen": 397279232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044423269809428287, + "loss": 1.4453, + "theoretical_loss": 4.014869598900496, + "tokens_seen": 397410304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044421263791374123, + "loss": 1.6041, + "theoretical_loss": 4.014721321513412, + "tokens_seen": 397541376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441925777331996, + "loss": 1.6268, + "theoretical_loss": 4.014573106689803, + "tokens_seen": 397672448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044417251755265796, + "loss": 1.5815, + "theoretical_loss": 4.014424954382663, + "tokens_seen": 397803520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441524573721163, + "loss": 1.4915, + "theoretical_loss": 4.014276864545036, + "tokens_seen": 397934592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044413239719157474, + "loss": 1.5733, + "theoretical_loss": 4.014128837130021, + "tokens_seen": 398065664 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.42839688062667847, + "objective/train/avg_token_score": 0.010799635201692581, + "objective/train/docs_used": 236468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.580099105834961, + "objective/train/original_loss": 3.822235107421875, + "objective/train/theoretical_loss": 4.0140548468163475, + "objective/train/tokens_used": 398131200, + "objective/train/unlikelihood_loss": 0.15448230504989624, + "theoretical_loss": 4.0140548468163475, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441123370110331, + "loss": 1.613, + "theoretical_loss": 4.0139808720907615, + "tokens_seen": 398196736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044409227683049146, + "loss": 1.4741, + "theoretical_loss": 4.013832969380456, + "tokens_seen": 398327808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440722166499498, + "loss": 1.4448, + "theoretical_loss": 4.013685128952353, + "tokens_seen": 398458880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044405215646940824, + "loss": 1.3947, + "theoretical_loss": 4.01353735075975, + "tokens_seen": 398589952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440320962888666, + "loss": 1.5525, + "theoretical_loss": 4.013389634755995, + "tokens_seen": 398721024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044401203610832497, + "loss": 1.5298, + "theoretical_loss": 4.013241980894488, + "tokens_seen": 398852096 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044399197592778333, + "loss": 1.4135, + "theoretical_loss": 4.013094389128677, + "tokens_seen": 398983168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439719157472417, + "loss": 1.5502, + "theoretical_loss": 4.012946859412062, + "tokens_seen": 399114240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439518555667001, + "loss": 1.5869, + "theoretical_loss": 4.012799391698191, + "tokens_seen": 399245312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044393179538615847, + "loss": 1.4827, + "theoretical_loss": 4.012651985940666, + "tokens_seen": 399376384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044391173520561683, + "loss": 1.6088, + "theoretical_loss": 4.012504642093133, + "tokens_seen": 399507456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438916750250752, + "loss": 1.4438, + "theoretical_loss": 4.012357360109293, + "tokens_seen": 399638528 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4299853444099426, + "objective/train/avg_token_score": 0.011655104346573353, + "objective/train/docs_used": 237759, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6880359649658203, + "objective/train/original_loss": 3.903928279876709, + "objective/train/theoretical_loss": 4.012210139942894, + "objective/train/tokens_used": 399769600, + "objective/train/unlikelihood_loss": 0.17748136818408966, + "theoretical_loss": 4.012210139942894, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438716148445336, + "loss": 1.6091, + "theoretical_loss": 4.012210139942894, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044385155466399197, + "loss": 1.6209, + "theoretical_loss": 4.012062981547736, + "tokens_seen": 399900672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044383149448345033, + "loss": 1.6198, + "theoretical_loss": 4.0119158848776655, + "tokens_seen": 400031744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438114343029087, + "loss": 1.3378, + "theoretical_loss": 4.011768849886582, + "tokens_seen": 400162816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437913741223671, + "loss": 1.6247, + "theoretical_loss": 4.011621876528431, + "tokens_seen": 400293888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437713139418255, + "loss": 1.7281, + "theoretical_loss": 4.011474964757211, + "tokens_seen": 400424960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044375125376128384, + "loss": 1.477, + "theoretical_loss": 4.011328114526968, + "tokens_seen": 400556032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437311935807422, + "loss": 1.6307, + "theoretical_loss": 4.011181325791797, + "tokens_seen": 400687104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044371113340020056, + "loss": 1.3736, + "theoretical_loss": 4.011034598505843, + "tokens_seen": 400818176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044369107321965903, + "loss": 1.4352, + "theoretical_loss": 4.010887932623299, + "tokens_seen": 400949248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436710130391174, + "loss": 1.6627, + "theoretical_loss": 4.01074132809841, + "tokens_seen": 401080320 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044365095285857576, + "loss": 1.6658, + "theoretical_loss": 4.010594784885466, + "tokens_seen": 401211392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436308926780341, + "loss": 1.4219, + "theoretical_loss": 4.01044830293881, + "tokens_seen": 401342464 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.31763195991516113, + "objective/train/avg_token_score": 0.008867236785590649, + "objective/train/docs_used": 238664, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1373664140701294, + "objective/train/original_loss": 3.9792494773864746, + "objective/train/theoretical_loss": 4.0103750849260855, + "objective/train/tokens_used": 401408000, + "objective/train/unlikelihood_loss": 0.17361101508140564, + "theoretical_loss": 4.0103750849260855, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044361083249749254, + "loss": 1.4158, + "theoretical_loss": 4.010301882212831, + "tokens_seen": 401473536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435907723169509, + "loss": 1.6034, + "theoretical_loss": 4.010155522661968, + "tokens_seen": 401604608 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044357071213640926, + "loss": 1.6155, + "theoretical_loss": 4.010009224240708, + "tokens_seen": 401735680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435506519558676, + "loss": 1.7323, + "theoretical_loss": 4.009862986903588, + "tokens_seen": 401866752 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443530591775326, + "loss": 1.5855, + "theoretical_loss": 4.009716810605193, + "tokens_seen": 401997824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435105315947844, + "loss": 1.481, + "theoretical_loss": 4.009570695300155, + "tokens_seen": 402128896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044349047141424276, + "loss": 1.5404, + "theoretical_loss": 4.009424640943157, + "tokens_seen": 402259968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434704112337011, + "loss": 1.5359, + "theoretical_loss": 4.00927864748893, + "tokens_seen": 402391040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434503510531595, + "loss": 1.5011, + "theoretical_loss": 4.009132714892251, + "tokens_seen": 402522112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434302908726179, + "loss": 1.6215, + "theoretical_loss": 4.008986843107949, + "tokens_seen": 402653184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044341023069207627, + "loss": 1.5534, + "theoretical_loss": 4.008841032090898, + "tokens_seen": 402784256 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044339017051153463, + "loss": 1.4099, + "theoretical_loss": 4.0086952817960215, + "tokens_seen": 402915328 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4825879633426666, + "objective/train/avg_token_score": 0.010442715138196945, + "objective/train/docs_used": 240054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6706886291503906, + "objective/train/original_loss": 3.7450501918792725, + "objective/train/theoretical_loss": 4.008549592178291, + "objective/train/tokens_used": 403046400, + "objective/train/unlikelihood_loss": 0.15135115385055542, + "theoretical_loss": 4.008549592178291, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443370110330993, + "loss": 1.4657, + "theoretical_loss": 4.008549592178291, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044335005015045135, + "loss": 1.4444, + "theoretical_loss": 4.008403963192725, + "tokens_seen": 403177472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044332998996990977, + "loss": 1.6658, + "theoretical_loss": 4.008258394794392, + "tokens_seen": 403308544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044330992978936813, + "loss": 1.5749, + "theoretical_loss": 4.008112886938409, + "tokens_seen": 403439616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432898696088265, + "loss": 1.4549, + "theoretical_loss": 4.007967439579936, + "tokens_seen": 403570688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044326980942828486, + "loss": 1.3103, + "theoretical_loss": 4.007822052674185, + "tokens_seen": 403701760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044324974924774327, + "loss": 1.6347, + "theoretical_loss": 4.007676726176415, + "tokens_seen": 403832832 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044322968906720163, + "loss": 1.3592, + "theoretical_loss": 4.007531460041933, + "tokens_seen": 403963904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044320962888666, + "loss": 1.4303, + "theoretical_loss": 4.00738625422609, + "tokens_seen": 404094976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044318956870611836, + "loss": 1.2863, + "theoretical_loss": 4.0072411086842905, + "tokens_seen": 404226048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431695085255767, + "loss": 1.4716, + "theoretical_loss": 4.007096023371981, + "tokens_seen": 404357120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044314944834503514, + "loss": 1.4291, + "theoretical_loss": 4.006950998244658, + "tokens_seen": 404488192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431293881644935, + "loss": 1.5443, + "theoretical_loss": 4.006806033257866, + "tokens_seen": 404619264 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4843597114086151, + "objective/train/avg_token_score": 0.03304527327418327, + "objective/train/docs_used": 240788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6086971759796143, + "objective/train/original_loss": 3.774994134902954, + "objective/train/theoretical_loss": 4.006733573303289, + "objective/train/tokens_used": 404684800, + "objective/train/unlikelihood_loss": 0.15561363101005554, + "theoretical_loss": 4.006733573303289, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044310932798395186, + "loss": 1.4442, + "theoretical_loss": 4.0066611283671945, + "tokens_seen": 404750336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430892678034102, + "loss": 1.498, + "theoretical_loss": 4.00651628352828, + "tokens_seen": 404881408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044306920762286864, + "loss": 1.5164, + "theoretical_loss": 4.00637149869681, + "tokens_seen": 405012480 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443049147442327, + "loss": 1.468, + "theoretical_loss": 4.006226773828514, + "tokens_seen": 405143552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044302908726178537, + "loss": 1.517, + "theoretical_loss": 4.0060821088791725, + "tokens_seen": 405274624 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044300902708124373, + "loss": 1.6745, + "theoretical_loss": 4.00593750380461, + "tokens_seen": 405405696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429889669007021, + "loss": 1.6433, + "theoretical_loss": 4.005792958560699, + "tokens_seen": 405536768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429689067201605, + "loss": 1.5431, + "theoretical_loss": 4.00564847310336, + "tokens_seen": 405667840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294884653961887, + "loss": 1.4441, + "theoretical_loss": 4.005504047388558, + "tokens_seen": 405798912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044292878635907723, + "loss": 1.4946, + "theoretical_loss": 4.005359681372306, + "tokens_seen": 405929984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429087261785356, + "loss": 1.5999, + "theoretical_loss": 4.005215375010662, + "tokens_seen": 406061056 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442888665997994, + "loss": 1.5072, + "theoretical_loss": 4.005071128259734, + "tokens_seen": 406192128 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.388868510723114, + "objective/train/avg_token_score": 0.012570223771035671, + "objective/train/docs_used": 241796, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4023349285125732, + "objective/train/original_loss": 3.9673821926116943, + "objective/train/theoretical_loss": 4.004926941075674, + "objective/train/tokens_used": 406323200, + "objective/train/unlikelihood_loss": 0.16696441173553467, + "theoretical_loss": 4.004926941075674, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044286860581745237, + "loss": 1.536, + "theoretical_loss": 4.004926941075674, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044284854563691073, + "loss": 1.482, + "theoretical_loss": 4.004782813414678, + "tokens_seen": 406454272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428284854563691, + "loss": 1.5833, + "theoretical_loss": 4.0046387452329935, + "tokens_seen": 406585344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428084252758275, + "loss": 1.5516, + "theoretical_loss": 4.00449473648691, + "tokens_seen": 406716416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427883650952859, + "loss": 1.5163, + "theoretical_loss": 4.004350787132766, + "tokens_seen": 406847488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044276830491474424, + "loss": 1.4299, + "theoretical_loss": 4.004206897126943, + "tokens_seen": 406978560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427482447342026, + "loss": 1.3877, + "theoretical_loss": 4.004063066425874, + "tokens_seen": 407109632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044272818455366096, + "loss": 1.5866, + "theoretical_loss": 4.003919294986032, + "tokens_seen": 407240704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427081243731194, + "loss": 1.3976, + "theoretical_loss": 4.003775582763939, + "tokens_seen": 407371776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044268806419257774, + "loss": 1.4836, + "theoretical_loss": 4.003631929716162, + "tokens_seen": 407502848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426680040120361, + "loss": 1.6081, + "theoretical_loss": 4.0034883357993145, + "tokens_seen": 407633920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044264794383149447, + "loss": 1.5082, + "theoretical_loss": 4.003344800970056, + "tokens_seen": 407764992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426278836509529, + "loss": 1.6963, + "theoretical_loss": 4.00320132518509, + "tokens_seen": 407896064 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.3863636255264282, + "objective/train/avg_token_score": 0.017943046987056732, + "objective/train/docs_used": 242638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4344366788864136, + "objective/train/original_loss": 4.310576915740967, + "objective/train/theoretical_loss": 4.0031296094207, + "objective/train/tokens_used": 407961600, + "objective/train/unlikelihood_loss": 0.1840338259935379, + "theoretical_loss": 4.0031296094207, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044260782347041124, + "loss": 1.5529, + "theoretical_loss": 4.003057908401168, + "tokens_seen": 408027136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425877632898696, + "loss": 1.5715, + "theoretical_loss": 4.002914550575085, + "tokens_seen": 408158208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044256770310932797, + "loss": 1.5628, + "theoretical_loss": 4.002771251663681, + "tokens_seen": 408289280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044254764292878633, + "loss": 1.4782, + "theoretical_loss": 4.0026280116238455, + "tokens_seen": 408420352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044252758274824475, + "loss": 1.6979, + "theoretical_loss": 4.0024848304125085, + "tokens_seen": 408551424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425075225677031, + "loss": 1.4791, + "theoretical_loss": 4.002341707986648, + "tokens_seen": 408682496 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044248746238716147, + "loss": 1.6751, + "theoretical_loss": 4.002198644303288, + "tokens_seen": 408813568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246740220661983, + "loss": 1.5135, + "theoretical_loss": 4.002055639319494, + "tokens_seen": 408944640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044244734202607825, + "loss": 1.6368, + "theoretical_loss": 4.001912692992379, + "tokens_seen": 409075712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424272818455366, + "loss": 1.5111, + "theoretical_loss": 4.001769805279103, + "tokens_seen": 409206784 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442407221664995, + "loss": 1.4949, + "theoretical_loss": 4.001626976136868, + "tokens_seen": 409337856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044238716148445334, + "loss": 1.5723, + "theoretical_loss": 4.001484205522922, + "tokens_seen": 409468928 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.42625856399536133, + "objective/train/avg_token_score": 0.018668072298169136, + "objective/train/docs_used": 244086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4982153177261353, + "objective/train/original_loss": 3.7382147312164307, + "objective/train/theoretical_loss": 4.001341493394558, + "objective/train/tokens_used": 409600000, + "objective/train/unlikelihood_loss": 0.17688016593456268, + "theoretical_loss": 4.001341493394558, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423671013039117, + "loss": 1.6406, + "theoretical_loss": 4.001341493394558, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423470411233701, + "loss": 1.59, + "theoretical_loss": 4.001198839709113, + "tokens_seen": 409731072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423269809428285, + "loss": 1.439, + "theoretical_loss": 4.001056244423971, + "tokens_seen": 409862144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230692076228684, + "loss": 1.5345, + "theoretical_loss": 4.000913707496556, + "tokens_seen": 409993216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422868605817452, + "loss": 1.5954, + "theoretical_loss": 4.000771228884344, + "tokens_seen": 410124288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422668004012036, + "loss": 1.6115, + "theoretical_loss": 4.000628808544848, + "tokens_seen": 410255360 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442246740220662, + "loss": 1.4791, + "theoretical_loss": 4.000486446435629, + "tokens_seen": 410386432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044222668004012034, + "loss": 1.4222, + "theoretical_loss": 4.000344142514294, + "tokens_seen": 410517504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422066198595787, + "loss": 1.3933, + "theoretical_loss": 4.000201896738492, + "tokens_seen": 410648576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421865596790371, + "loss": 1.6559, + "theoretical_loss": 4.0000597090659165, + "tokens_seen": 410779648 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044216649949849554, + "loss": 1.5433, + "theoretical_loss": 3.9999175794543063, + "tokens_seen": 410910720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421464393179539, + "loss": 1.4679, + "theoretical_loss": 3.999775507861443, + "tokens_seen": 411041792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044212637913741226, + "loss": 1.3713, + "theoretical_loss": 3.9996334942451544, + "tokens_seen": 411172864 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.39326736330986023, + "objective/train/avg_token_score": 0.010529226623475552, + "objective/train/docs_used": 244630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.416571021080017, + "objective/train/original_loss": 3.8771302700042725, + "objective/train/theoretical_loss": 3.999562509165058, + "objective/train/tokens_used": 411238400, + "objective/train/unlikelihood_loss": 0.1982622891664505, + "theoretical_loss": 3.999562509165058, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421063189568706, + "loss": 1.6983, + "theoretical_loss": 3.9994915385633103, + "tokens_seen": 411303936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044208625877632904, + "loss": 1.5469, + "theoretical_loss": 3.9993496407738256, + "tokens_seen": 411435008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420661985957874, + "loss": 1.3864, + "theoretical_loss": 3.999207800834659, + "tokens_seen": 411566080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044204613841524577, + "loss": 1.5093, + "theoretical_loss": 3.999066018703814, + "tokens_seen": 411697152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044202607823470413, + "loss": 1.617, + "theoretical_loss": 3.9989242943393357, + "tokens_seen": 411828224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420060180541625, + "loss": 1.5732, + "theoretical_loss": 3.998782627699315, + "tokens_seen": 411959296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419859578736209, + "loss": 1.657, + "theoretical_loss": 3.9986410187418855, + "tokens_seen": 412090368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044196589769307927, + "loss": 1.5478, + "theoretical_loss": 3.9984994674252254, + "tokens_seen": 412221440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044194583751253763, + "loss": 1.4761, + "theoretical_loss": 3.9983579737075554, + "tokens_seen": 412352512 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441925777331996, + "loss": 1.4249, + "theoretical_loss": 3.9982165375471403, + "tokens_seen": 412483584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419057171514544, + "loss": 1.5888, + "theoretical_loss": 3.9980751589022887, + "tokens_seen": 412614656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044188565697091277, + "loss": 1.4857, + "theoretical_loss": 3.9979338377313525, + "tokens_seen": 412745728 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4978005886077881, + "objective/train/avg_token_score": 0.016746150329709053, + "objective/train/docs_used": 245807, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8077481985092163, + "objective/train/original_loss": 4.005298614501953, + "objective/train/theoretical_loss": 3.997792573992726, + "objective/train/tokens_used": 412876800, + "objective/train/unlikelihood_loss": 0.1058061420917511, + "theoretical_loss": 3.997792573992726, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044186559679037113, + "loss": 1.5612, + "theoretical_loss": 3.997792573992726, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418455366098295, + "loss": 1.3567, + "theoretical_loss": 3.997651367644848, + "tokens_seen": 413007872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418254764292879, + "loss": 1.674, + "theoretical_loss": 3.9975102186462004, + "tokens_seen": 413138944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418054162487463, + "loss": 1.4358, + "theoretical_loss": 3.9973691269553076, + "tokens_seen": 413270016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044178535606820464, + "loss": 1.4689, + "theoretical_loss": 3.9972280925307384, + "tokens_seen": 413401088 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441765295887663, + "loss": 1.6083, + "theoretical_loss": 3.9970871153311025, + "tokens_seen": 413532160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044174523570712136, + "loss": 1.434, + "theoretical_loss": 3.996946195315055, + "tokens_seen": 413663232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417251755265798, + "loss": 1.6717, + "theoretical_loss": 3.996805332441293, + "tokens_seen": 413794304 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044170511534603814, + "loss": 1.5077, + "theoretical_loss": 3.996664526668555, + "tokens_seen": 413925376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416850551654965, + "loss": 1.5328, + "theoretical_loss": 3.996523777955626, + "tokens_seen": 414056448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044166499498495487, + "loss": 1.4311, + "theoretical_loss": 3.9963830862613303, + "tokens_seen": 414187520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416449348044133, + "loss": 1.5412, + "theoretical_loss": 3.9962424515445356, + "tokens_seen": 414318592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044162487462387164, + "loss": 1.4443, + "theoretical_loss": 3.9961018737641547, + "tokens_seen": 414449664 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4602883756160736, + "objective/train/avg_token_score": 0.015043980441987514, + "objective/train/docs_used": 246540, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.704790472984314, + "objective/train/original_loss": 4.011831283569336, + "objective/train/theoretical_loss": 3.99603160621229, + "objective/train/tokens_used": 414515200, + "objective/train/unlikelihood_loss": 0.13677598536014557, + "theoretical_loss": 3.99603160621229, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044160481444333, + "loss": 1.4581, + "theoretical_loss": 3.9959613528791396, + "tokens_seen": 414580736 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044158475426278837, + "loss": 1.4903, + "theoretical_loss": 3.9958208888484865, + "tokens_seen": 414711808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044156469408224673, + "loss": 1.5552, + "theoretical_loss": 3.9956804816312355, + "tokens_seen": 414842880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044154463390170515, + "loss": 1.5021, + "theoretical_loss": 3.9955401311864662, + "tokens_seen": 414973952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415245737211635, + "loss": 1.5002, + "theoretical_loss": 3.995399837473302, + "tokens_seen": 415105024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044150451354062187, + "loss": 1.5115, + "theoretical_loss": 3.99525960045091, + "tokens_seen": 415236096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044148445336008023, + "loss": 1.5434, + "theoretical_loss": 3.9951194200784976, + "tokens_seen": 415367168 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044146439317953865, + "loss": 1.5491, + "theoretical_loss": 3.9949792963153143, + "tokens_seen": 415498240 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441444332998997, + "loss": 1.5161, + "theoretical_loss": 3.994839229120654, + "tokens_seen": 415629312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414242728184554, + "loss": 1.5123, + "theoretical_loss": 3.99469921845385, + "tokens_seen": 415760384 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044140421263791374, + "loss": 1.4513, + "theoretical_loss": 3.9945592642742795, + "tokens_seen": 415891456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413841524573721, + "loss": 1.5792, + "theoretical_loss": 3.9944193665413605, + "tokens_seen": 416022528 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.38825756311416626, + "objective/train/avg_token_score": 0.025925040245056152, + "objective/train/docs_used": 247824, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4125205278396606, + "objective/train/original_loss": 3.7366883754730225, + "objective/train/theoretical_loss": 3.994279525214554, + "objective/train/tokens_used": 416153600, + "objective/train/unlikelihood_loss": 0.2123020440340042, + "theoretical_loss": 3.994279525214554, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413640922768305, + "loss": 1.4642, + "theoretical_loss": 3.994279525214554, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413440320962889, + "loss": 1.6119, + "theoretical_loss": 3.994139740253363, + "tokens_seen": 416284672 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044132397191574724, + "loss": 1.4493, + "theoretical_loss": 3.99400001161733, + "tokens_seen": 416415744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413039117352056, + "loss": 1.4857, + "theoretical_loss": 3.9938603392660426, + "tokens_seen": 416546816 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441283851554664, + "loss": 1.5087, + "theoretical_loss": 3.993720723159127, + "tokens_seen": 416677888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412637913741224, + "loss": 1.5877, + "theoretical_loss": 3.9935811632562532, + "tokens_seen": 416808960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044124373119358074, + "loss": 1.5386, + "theoretical_loss": 3.9934416595171323, + "tokens_seen": 416940032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412236710130391, + "loss": 1.5819, + "theoretical_loss": 3.993302211901516, + "tokens_seen": 417071104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044120361083249747, + "loss": 1.5589, + "theoretical_loss": 3.993162820369199, + "tokens_seen": 417202176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411835506519559, + "loss": 1.5473, + "theoretical_loss": 3.993023484880016, + "tokens_seen": 417333248 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044116349047141425, + "loss": 1.5015, + "theoretical_loss": 3.9928842053938443, + "tokens_seen": 417464320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411434302908726, + "loss": 1.3862, + "theoretical_loss": 3.9927449818706005, + "tokens_seen": 417595392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044112337011033097, + "loss": 1.5006, + "theoretical_loss": 3.9926058142702456, + "tokens_seen": 417726464 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4471529722213745, + "objective/train/avg_token_score": 0.010523061268031597, + "objective/train/docs_used": 248559, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6317007541656494, + "objective/train/original_loss": 3.7148232460021973, + "objective/train/theoretical_loss": 3.9925362514286498, + "objective/train/tokens_used": 417792000, + "objective/train/unlikelihood_loss": 0.19979330897331238, + "theoretical_loss": 3.9925362514286498, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411033099297894, + "loss": 1.4831, + "theoretical_loss": 3.992466702552779, + "tokens_seen": 417857536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044108324974924775, + "loss": 1.5975, + "theoretical_loss": 3.9923276466782425, + "tokens_seen": 417988608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410631895687061, + "loss": 1.6912, + "theoretical_loss": 3.992188646606719, + "tokens_seen": 418119680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410431293881645, + "loss": 1.4185, + "theoretical_loss": 3.992049702298332, + "tokens_seen": 418250752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044102306920762284, + "loss": 1.4789, + "theoretical_loss": 3.991910813713246, + "tokens_seen": 418381824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044100300902708125, + "loss": 1.4325, + "theoretical_loss": 3.991771980811667, + "tokens_seen": 418512896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409829488465396, + "loss": 1.6477, + "theoretical_loss": 3.9916332035538415, + "tokens_seen": 418643968 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440962888665998, + "loss": 1.6004, + "theoretical_loss": 3.991494481900056, + "tokens_seen": 418775040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094282848545634, + "loss": 1.4793, + "theoretical_loss": 3.99135581581064, + "tokens_seen": 418906112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044092276830491476, + "loss": 1.5306, + "theoretical_loss": 3.9912172052459614, + "tokens_seen": 419037184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409027081243731, + "loss": 1.6564, + "theoretical_loss": 3.991078650166429, + "tokens_seen": 419168256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408826479438315, + "loss": 1.5377, + "theoretical_loss": 3.990940150532494, + "tokens_seen": 419299328 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3536168038845062, + "objective/train/avg_token_score": 0.01034875400364399, + "objective/train/docs_used": 249785, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2462135553359985, + "objective/train/original_loss": 4.00803804397583, + "objective/train/theoretical_loss": 3.990801706304647, + "objective/train/tokens_used": 419430400, + "objective/train/unlikelihood_loss": 0.17689494788646698, + "theoretical_loss": 3.990801706304647, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086258776328984, + "loss": 1.5763, + "theoretical_loss": 3.990801706304647, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408425275827482, + "loss": 1.4898, + "theoretical_loss": 3.990663317443418, + "tokens_seen": 419561472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408224674022066, + "loss": 1.5875, + "theoretical_loss": 3.9905249839093786, + "tokens_seen": 419692544 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440802407221665, + "loss": 1.5456, + "theoretical_loss": 3.990386705663142, + "tokens_seen": 419823616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044078234704112335, + "loss": 1.6324, + "theoretical_loss": 3.990248482665359, + "tokens_seen": 419954688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407622868605817, + "loss": 1.6974, + "theoretical_loss": 3.9901103148767225, + "tokens_seen": 420085760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407422266800401, + "loss": 1.5273, + "theoretical_loss": 3.989972202257966, + "tokens_seen": 420216832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407221664994985, + "loss": 1.321, + "theoretical_loss": 3.989834144769861, + "tokens_seen": 420347904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044070210631895685, + "loss": 1.5629, + "theoretical_loss": 3.989696142373221, + "tokens_seen": 420478976 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044068204613841527, + "loss": 1.6409, + "theoretical_loss": 3.9895581950288994, + "tokens_seen": 420610048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406619859578737, + "loss": 1.4717, + "theoretical_loss": 3.989420302697789, + "tokens_seen": 420741120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044064192577733204, + "loss": 1.5218, + "theoretical_loss": 3.989282465340823, + "tokens_seen": 420872192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406218655967904, + "loss": 1.4617, + "theoretical_loss": 3.9891446829189734, + "tokens_seen": 421003264 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4630987346172333, + "objective/train/avg_token_score": 0.02803061157464981, + "objective/train/docs_used": 250392, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7076679468154907, + "objective/train/original_loss": 4.153679370880127, + "objective/train/theoretical_loss": 3.989075812296533, + "objective/train/tokens_used": 421068800, + "objective/train/unlikelihood_loss": 0.13347437977790833, + "theoretical_loss": 3.989075812296533, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044060180541624877, + "loss": 1.6244, + "theoretical_loss": 3.989006955393254, + "tokens_seen": 421134336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044058174523570713, + "loss": 1.7096, + "theoretical_loss": 3.9888692827247167, + "tokens_seen": 421265408 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044056168505516555, + "loss": 1.4498, + "theoretical_loss": 3.988731664874454, + "tokens_seen": 421396480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405416248746239, + "loss": 1.3687, + "theoretical_loss": 3.988594101803598, + "tokens_seen": 421527552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044052156469408227, + "loss": 1.687, + "theoretical_loss": 3.9884565934733196, + "tokens_seen": 421658624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044050150451354063, + "loss": 1.444, + "theoretical_loss": 3.9883191398448306, + "tokens_seen": 421789696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044048144433299905, + "loss": 1.3943, + "theoretical_loss": 3.988181740879382, + "tokens_seen": 421920768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404613841524574, + "loss": 1.4322, + "theoretical_loss": 3.988044396538263, + "tokens_seen": 422051840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404413239719158, + "loss": 1.5675, + "theoretical_loss": 3.9879071067828034, + "tokens_seen": 422182912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044042126379137414, + "loss": 1.5548, + "theoretical_loss": 3.987769871574373, + "tokens_seen": 422313984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404012036108325, + "loss": 1.4586, + "theoretical_loss": 3.987632690874379, + "tokens_seen": 422445056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403811434302909, + "loss": 1.6742, + "theoretical_loss": 3.98749556464427, + "tokens_seen": 422576128 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3960777223110199, + "objective/train/avg_token_score": 0.019322294741868973, + "objective/train/docs_used": 251199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4102435111999512, + "objective/train/original_loss": 4.012388706207275, + "objective/train/theoretical_loss": 3.987358492845532, + "objective/train/tokens_used": 422707200, + "objective/train/unlikelihood_loss": 0.17995640635490417, + "theoretical_loss": 3.987358492845532, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403610832497493, + "loss": 1.4228, + "theoretical_loss": 3.987358492845532, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044034102306920764, + "loss": 1.4906, + "theoretical_loss": 3.987221475439691, + "tokens_seen": 422838272 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440320962888666, + "loss": 1.3949, + "theoretical_loss": 3.9870845123883116, + "tokens_seen": 422969344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403009027081244, + "loss": 1.3426, + "theoretical_loss": 3.9869476036529994, + "tokens_seen": 423100416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402808425275828, + "loss": 1.562, + "theoretical_loss": 3.986810749195396, + "tokens_seen": 423231488 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044026078234704114, + "loss": 1.8139, + "theoretical_loss": 3.9866739489771845, + "tokens_seen": 423362560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402407221664995, + "loss": 1.6235, + "theoretical_loss": 3.986537202960085, + "tokens_seen": 423493632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044022066198595787, + "loss": 1.6967, + "theoretical_loss": 3.986400511105858, + "tokens_seen": 423624704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402006018054163, + "loss": 1.3629, + "theoretical_loss": 3.986263873376301, + "tokens_seen": 423755776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044018054162487465, + "loss": 1.6447, + "theoretical_loss": 3.9861272897332527, + "tokens_seen": 423886848 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440160481444333, + "loss": 1.5283, + "theoretical_loss": 3.9859907601385887, + "tokens_seen": 424017920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044014042126379137, + "loss": 1.4187, + "theoretical_loss": 3.985854284554223, + "tokens_seen": 424148992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401203610832498, + "loss": 1.6492, + "theoretical_loss": 3.98571786294211, + "tokens_seen": 424280064 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4866202473640442, + "objective/train/avg_token_score": 0.008693613111972809, + "objective/train/docs_used": 252247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6004599332809448, + "objective/train/original_loss": 3.3173439502716064, + "objective/train/theoretical_loss": 3.9856496723637687, + "objective/train/tokens_used": 424345600, + "objective/train/unlikelihood_loss": 0.19723816215991974, + "theoretical_loss": 3.9856496723637687, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044010030090270815, + "loss": 1.6577, + "theoretical_loss": 3.985581495264241, + "tokens_seen": 424411136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400802407221665, + "loss": 1.5225, + "theoretical_loss": 3.9854451814826457, + "tokens_seen": 424542208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400601805416249, + "loss": 1.4646, + "theoretical_loss": 3.985308921559394, + "tokens_seen": 424673280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044004012036108324, + "loss": 1.4294, + "theoretical_loss": 3.9851727154565926, + "tokens_seen": 424804352 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044002006018054165, + "loss": 1.6139, + "theoretical_loss": 3.9850365631363873, + "tokens_seen": 424935424 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044, + "loss": 1.5823, + "theoretical_loss": 3.984900464560961, + "tokens_seen": 425066496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399799398194584, + "loss": 1.6172, + "theoretical_loss": 3.984764419692537, + "tokens_seen": 425197568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043995987963891674, + "loss": 1.6824, + "theoretical_loss": 3.984628428493374, + "tokens_seen": 425328640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043993981945837516, + "loss": 1.5701, + "theoretical_loss": 3.984492490925772, + "tokens_seen": 425459712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399197592778335, + "loss": 1.5669, + "theoretical_loss": 3.9843566069520664, + "tokens_seen": 425590784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398996990972919, + "loss": 1.3379, + "theoretical_loss": 3.984220776534632, + "tokens_seen": 425721856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043987963891675024, + "loss": 1.6932, + "theoretical_loss": 3.9840849996358814, + "tokens_seen": 425852928 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.35037878155708313, + "objective/train/avg_token_score": 0.010006745345890522, + "objective/train/docs_used": 253004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2638418674468994, + "objective/train/original_loss": 3.8855535984039307, + "objective/train/theoretical_loss": 3.9839492762182647, + "objective/train/tokens_used": 425984000, + "objective/train/unlikelihood_loss": 0.18474146723747253, + "theoretical_loss": 3.9839492762182647, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398595787362086, + "loss": 1.5895, + "theoretical_loss": 3.9839492762182647, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439839518555667, + "loss": 1.5817, + "theoretical_loss": 3.9838136062442704, + "tokens_seen": 426115072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398194583751254, + "loss": 1.6915, + "theoretical_loss": 3.9836779896764254, + "tokens_seen": 426246144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043979939819458375, + "loss": 1.7129, + "theoretical_loss": 3.983542426477292, + "tokens_seen": 426377216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397793380140421, + "loss": 1.6312, + "theoretical_loss": 3.983406916609473, + "tokens_seen": 426508288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397592778335005, + "loss": 1.5439, + "theoretical_loss": 3.983271460035608, + "tokens_seen": 426639360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397392176529589, + "loss": 1.5743, + "theoretical_loss": 3.983136056718373, + "tokens_seen": 426770432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043971915747241725, + "loss": 1.5065, + "theoretical_loss": 3.9830007066204836, + "tokens_seen": 426901504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396990972918756, + "loss": 1.5345, + "theoretical_loss": 3.982865409704692, + "tokens_seen": 427032576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043967903711133403, + "loss": 1.6371, + "theoretical_loss": 3.9827301659337864, + "tokens_seen": 427163648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396589769307924, + "loss": 1.626, + "theoretical_loss": 3.982594975270595, + "tokens_seen": 427294720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043963891675025075, + "loss": 1.5697, + "theoretical_loss": 3.9824598376779825, + "tokens_seen": 427425792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396188565697091, + "loss": 1.4803, + "theoretical_loss": 3.98232475311885, + "tokens_seen": 427556864 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.33889296650886536, + "objective/train/avg_token_score": 0.04264990985393524, + "objective/train/docs_used": 254260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.204171061515808, + "objective/train/original_loss": 4.482820987701416, + "objective/train/theoretical_loss": 3.9822572307152564, + "objective/train/tokens_used": 427622400, + "objective/train/unlikelihood_loss": 0.09470682591199875, + "theoretical_loss": 3.9822572307152564, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395987963891675, + "loss": 1.5879, + "theoretical_loss": 3.9821897215561375, + "tokens_seen": 427687936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395787362086259, + "loss": 1.4706, + "theoretical_loss": 3.98205474295282, + "tokens_seen": 427819008 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043955867602808426, + "loss": 1.558, + "theoretical_loss": 3.9819198172719124, + "tokens_seen": 427950080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395386158475426, + "loss": 1.6488, + "theoretical_loss": 3.9817849444764644, + "tokens_seen": 428081152 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439518555667001, + "loss": 1.641, + "theoretical_loss": 3.9816501245295646, + "tokens_seen": 428212224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394984954864594, + "loss": 1.6366, + "theoretical_loss": 3.981515357394338, + "tokens_seen": 428343296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043947843530591776, + "loss": 1.3471, + "theoretical_loss": 3.981380643033945, + "tokens_seen": 428474368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394583751253761, + "loss": 1.5653, + "theoretical_loss": 3.981245981411586, + "tokens_seen": 428605440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394383149448345, + "loss": 1.573, + "theoretical_loss": 3.981111372490496, + "tokens_seen": 428736512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043941825476429285, + "loss": 1.6642, + "theoretical_loss": 3.9809768162339476, + "tokens_seen": 428867584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043939819458375126, + "loss": 1.5006, + "theoretical_loss": 3.9808423126052506, + "tokens_seen": 428998656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393781344032096, + "loss": 1.4711, + "theoretical_loss": 3.9807078615677502, + "tokens_seen": 429129728 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4456256031990051, + "objective/train/avg_token_score": 0.0028587188571691513, + "objective/train/docs_used": 255066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7612452507019043, + "objective/train/original_loss": 4.201564311981201, + "objective/train/theoretical_loss": 3.9805734630848306, + "objective/train/tokens_used": 429260800, + "objective/train/unlikelihood_loss": 0.11549005657434464, + "theoretical_loss": 3.9805734630848306, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439358074222668, + "loss": 1.6208, + "theoretical_loss": 3.9805734630848306, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933801404212635, + "loss": 1.3378, + "theoretical_loss": 3.9804391171199103, + "tokens_seen": 429391872 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043931795386158477, + "loss": 1.4226, + "theoretical_loss": 3.9803048236364456, + "tokens_seen": 429522944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043929789368104313, + "loss": 1.437, + "theoretical_loss": 3.98017058259793, + "tokens_seen": 429654016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392778335005015, + "loss": 1.6993, + "theoretical_loss": 3.9800363939678918, + "tokens_seen": 429785088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043925777331995985, + "loss": 1.454, + "theoretical_loss": 3.979902257709897, + "tokens_seen": 429916160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392377131394182, + "loss": 1.5987, + "theoretical_loss": 3.979768173787548, + "tokens_seen": 430047232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921765295887663, + "loss": 1.7553, + "theoretical_loss": 3.9796341421644827, + "tokens_seen": 430178304 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439197592778335, + "loss": 1.5489, + "theoretical_loss": 3.979500162804377, + "tokens_seen": 430309376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391775325977934, + "loss": 1.4274, + "theoretical_loss": 3.9793662356709407, + "tokens_seen": 430440448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043915747241725177, + "loss": 1.6129, + "theoretical_loss": 3.9792323607279223, + "tokens_seen": 430571520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391374122367102, + "loss": 1.4735, + "theoretical_loss": 3.9790985379391053, + "tokens_seen": 430702592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043911735205616855, + "loss": 1.4495, + "theoretical_loss": 3.978964767268309, + "tokens_seen": 430833664 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.35037878155708313, + "objective/train/avg_token_score": 0.03396159037947655, + "objective/train/docs_used": 256338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.339438557624817, + "objective/train/original_loss": 4.461732387542725, + "objective/train/theoretical_loss": 3.9788979014658716, + "objective/train/tokens_used": 430899200, + "objective/train/unlikelihood_loss": 0.13601770997047424, + "theoretical_loss": 3.9788979014658716, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390972918756269, + "loss": 1.6812, + "theoretical_loss": 3.978831048679389, + "tokens_seen": 430964736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390772316950853, + "loss": 1.5037, + "theoretical_loss": 3.9786973821362377, + "tokens_seen": 431095808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043905717151454364, + "loss": 1.3236, + "theoretical_loss": 3.978563767602782, + "tokens_seen": 431226880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043903711133400205, + "loss": 1.7428, + "theoretical_loss": 3.9784302050429874, + "tokens_seen": 431357952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390170511534604, + "loss": 1.2738, + "theoretical_loss": 3.9782966944208518, + "tokens_seen": 431489024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389969909729188, + "loss": 1.6967, + "theoretical_loss": 3.9781632357004124, + "tokens_seen": 431620096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043897693079237714, + "loss": 1.6041, + "theoretical_loss": 3.9780298288457394, + "tokens_seen": 431751168 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043895687061183556, + "loss": 1.4963, + "theoretical_loss": 3.977896473820941, + "tokens_seen": 431882240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389368104312939, + "loss": 1.4479, + "theoretical_loss": 3.977763170590159, + "tokens_seen": 432013312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389167502507523, + "loss": 1.5352, + "theoretical_loss": 3.9776299191175726, + "tokens_seen": 432144384 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043889669007021064, + "loss": 1.6516, + "theoretical_loss": 3.977496719367396, + "tokens_seen": 432275456 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438876629889669, + "loss": 1.6093, + "theoretical_loss": 3.9773635713038784, + "tokens_seen": 432406528 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.46212121844291687, + "objective/train/avg_token_score": 0.020001525059342384, + "objective/train/docs_used": 257001, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5973520278930664, + "objective/train/original_loss": 3.78364634513855, + "objective/train/theoretical_loss": 3.9772304748913054, + "objective/train/tokens_used": 432537600, + "objective/train/unlikelihood_loss": 0.15700258314609528, + "theoretical_loss": 3.9772304748913054, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388565697091274, + "loss": 1.5176, + "theoretical_loss": 3.9772304748913054, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388365095285858, + "loss": 1.5094, + "theoretical_loss": 3.977097430093999, + "tokens_seen": 432668672 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043881644934804415, + "loss": 1.5875, + "theoretical_loss": 3.9769644368763135, + "tokens_seen": 432799744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387963891675025, + "loss": 1.5692, + "theoretical_loss": 3.9768314952026422, + "tokens_seen": 432930816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387763289869609, + "loss": 1.4838, + "theoretical_loss": 3.976698605037411, + "tokens_seen": 433061888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387562688064193, + "loss": 1.4281, + "theoretical_loss": 3.976565766345083, + "tokens_seen": 433192960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043873620862587765, + "loss": 1.4939, + "theoretical_loss": 3.976432979090155, + "tokens_seen": 433324032 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438716148445336, + "loss": 1.5756, + "theoretical_loss": 3.9763002432371604, + "tokens_seen": 433455104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043869608826479443, + "loss": 1.6247, + "theoretical_loss": 3.9761675587506673, + "tokens_seen": 433586176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386760280842528, + "loss": 1.4458, + "theoretical_loss": 3.976034925595278, + "tokens_seen": 433717248 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043865596790371115, + "loss": 1.7759, + "theoretical_loss": 3.9759023437356316, + "tokens_seen": 433848320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386359077231695, + "loss": 1.5281, + "theoretical_loss": 3.9757698131364005, + "tokens_seen": 433979392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386158475426279, + "loss": 1.6324, + "theoretical_loss": 3.9756373337622932, + "tokens_seen": 434110464 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4903470277786255, + "objective/train/avg_token_score": 0.012690459378063679, + "objective/train/docs_used": 257982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6558393239974976, + "objective/train/original_loss": 3.627277374267578, + "objective/train/theoretical_loss": 3.975571113273642, + "objective/train/tokens_used": 434176000, + "objective/train/unlikelihood_loss": 0.18798591196537018, + "theoretical_loss": 3.975571113273642, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385957873620863, + "loss": 1.5611, + "theoretical_loss": 3.975504905578053, + "tokens_seen": 434241536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043857572718154466, + "loss": 1.6058, + "theoretical_loss": 3.9753725285484576, + "tokens_seen": 434372608 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438555667001003, + "loss": 1.6332, + "theoretical_loss": 3.9752402026383207, + "tokens_seen": 434503680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385356068204614, + "loss": 1.6041, + "theoretical_loss": 3.9751079278124886, + "tokens_seen": 434634752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385155466399198, + "loss": 1.4406, + "theoretical_loss": 3.9749757040358453, + "tokens_seen": 434765824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043849548645937816, + "loss": 1.5341, + "theoretical_loss": 3.974843531273307, + "tokens_seen": 434896896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384754262788365, + "loss": 1.5085, + "theoretical_loss": 3.974711409489826, + "tokens_seen": 435027968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384553660982949, + "loss": 1.3682, + "theoretical_loss": 3.9745793386503885, + "tokens_seen": 435159040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043843530591775325, + "loss": 1.5537, + "theoretical_loss": 3.9744473187200158, + "tokens_seen": 435290112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043841524573721166, + "loss": 1.5423, + "theoretical_loss": 3.974315349663763, + "tokens_seen": 435421184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043839518555667, + "loss": 1.5619, + "theoretical_loss": 3.974183431446721, + "tokens_seen": 435552256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383751253761284, + "loss": 1.5113, + "theoretical_loss": 3.9740515640340135, + "tokens_seen": 435683328 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3175097703933716, + "objective/train/avg_token_score": 0.01103399321436882, + "objective/train/docs_used": 258654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1034122705459595, + "objective/train/original_loss": 3.728519916534424, + "objective/train/theoretical_loss": 3.973919747390801, + "objective/train/tokens_used": 435814400, + "objective/train/unlikelihood_loss": 0.20263297855854034, + "theoretical_loss": 3.973919747390801, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043835506519558675, + "loss": 1.4324, + "theoretical_loss": 3.973919747390801, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043833500501504517, + "loss": 1.7232, + "theoretical_loss": 3.9737879814822747, + "tokens_seen": 435945472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043831494483450353, + "loss": 1.5674, + "theoretical_loss": 3.9736562662736636, + "tokens_seen": 436076544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382948846539619, + "loss": 1.6733, + "theoretical_loss": 3.9735246017302295, + "tokens_seen": 436207616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043827482447342025, + "loss": 1.6952, + "theoretical_loss": 3.9733929878172685, + "tokens_seen": 436338688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382547642928786, + "loss": 1.6721, + "theoretical_loss": 3.9732614245001106, + "tokens_seen": 436469760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043823470411233703, + "loss": 1.4846, + "theoretical_loss": 3.9731299117441203, + "tokens_seen": 436600832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382146439317954, + "loss": 1.4607, + "theoretical_loss": 3.972998449514697, + "tokens_seen": 436731904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043819458375125376, + "loss": 1.5541, + "theoretical_loss": 3.972867037777273, + "tokens_seen": 436862976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381745235707121, + "loss": 1.5188, + "theoretical_loss": 3.9727356764973143, + "tokens_seen": 436994048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043815446339017053, + "loss": 1.5887, + "theoretical_loss": 3.972604365640322, + "tokens_seen": 437125120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381344032096289, + "loss": 1.5462, + "theoretical_loss": 3.97247310517183, + "tokens_seen": 437256192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043811434302908726, + "loss": 1.6721, + "theoretical_loss": 3.972341895057408, + "tokens_seen": 437387264 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3990102708339691, + "objective/train/avg_token_score": 0.014954428188502789, + "objective/train/docs_used": 259916, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3668731451034546, + "objective/train/original_loss": 3.7425804138183594, + "objective/train/theoretical_loss": 3.972276308872222, + "objective/train/tokens_used": 437452800, + "objective/train/unlikelihood_loss": 0.18754182755947113, + "theoretical_loss": 3.972276308872222, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380942828485456, + "loss": 1.5532, + "theoretical_loss": 3.972210735262657, + "tokens_seen": 437518336 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438074222668004, + "loss": 1.7842, + "theoretical_loss": 3.9720796257532136, + "tokens_seen": 437649408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380541624874624, + "loss": 1.5287, + "theoretical_loss": 3.9719485664947483, + "tokens_seen": 437780480 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043803410230692076, + "loss": 1.6341, + "theoretical_loss": 3.9718175574529635, + "tokens_seen": 437911552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380140421263791, + "loss": 1.4245, + "theoretical_loss": 3.971686598593596, + "tokens_seen": 438042624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379939819458375, + "loss": 1.5868, + "theoretical_loss": 3.9715556898824182, + "tokens_seen": 438173696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379739217652959, + "loss": 1.6966, + "theoretical_loss": 3.9714248312852343, + "tokens_seen": 438304768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043795386158475427, + "loss": 1.494, + "theoretical_loss": 3.9712940227678803, + "tokens_seen": 438435840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043793380140421263, + "loss": 1.6852, + "theoretical_loss": 3.971163264296229, + "tokens_seen": 438566912 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437913741223671, + "loss": 1.4822, + "theoretical_loss": 3.9710325558361856, + "tokens_seen": 438697984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043789368104312935, + "loss": 1.6556, + "theoretical_loss": 3.970901897353688, + "tokens_seen": 438829056 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043787362086258777, + "loss": 1.7292, + "theoretical_loss": 3.9707712888147073, + "tokens_seen": 438960128 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3905791640281677, + "objective/train/avg_token_score": 0.019270719960331917, + "objective/train/docs_used": 260524, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4323198795318604, + "objective/train/original_loss": 4.0027384757995605, + "objective/train/theoretical_loss": 3.9706407301852487, + "objective/train/tokens_used": 439091200, + "objective/train/unlikelihood_loss": 0.153401181101799, + "theoretical_loss": 3.9706407301852487, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785356068204613, + "loss": 1.6313, + "theoretical_loss": 3.9706407301852487, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378335005015045, + "loss": 1.6079, + "theoretical_loss": 3.9705102214313506, + "tokens_seen": 439222272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781344032096286, + "loss": 1.5786, + "theoretical_loss": 3.970379762519084, + "tokens_seen": 439353344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043779338014042127, + "loss": 1.6811, + "theoretical_loss": 3.9702493534145544, + "tokens_seen": 439484416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777331995987963, + "loss": 1.5142, + "theoretical_loss": 3.970118994083899, + "tokens_seen": 439615488 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437753259779338, + "loss": 1.5964, + "theoretical_loss": 3.969988684493289, + "tokens_seen": 439746560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043773319959879636, + "loss": 1.4616, + "theoretical_loss": 3.969858424608928, + "tokens_seen": 439877632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377131394182547, + "loss": 1.6263, + "theoretical_loss": 3.969728214397053, + "tokens_seen": 440008704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043769307923771314, + "loss": 1.5717, + "theoretical_loss": 3.969598053823934, + "tokens_seen": 440139776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043767301905717155, + "loss": 1.5974, + "theoretical_loss": 3.969467942855875, + "tokens_seen": 440270848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376529588766299, + "loss": 1.5539, + "theoretical_loss": 3.9693378814592104, + "tokens_seen": 440401920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376328986960883, + "loss": 1.6919, + "theoretical_loss": 3.969207869600309, + "tokens_seen": 440532992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376128385155467, + "loss": 1.5798, + "theoretical_loss": 3.9690779072455733, + "tokens_seen": 440664064 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4020650088787079, + "objective/train/avg_token_score": 0.015384318307042122, + "objective/train/docs_used": 261739, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4516440629959106, + "objective/train/original_loss": 3.8263213634490967, + "objective/train/theoretical_loss": 3.969012944621776, + "objective/train/tokens_used": 440729600, + "objective/train/unlikelihood_loss": 0.15117530524730682, + "theoretical_loss": 3.969012944621776, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043759277833500506, + "loss": 1.5678, + "theoretical_loss": 3.968947994361436, + "tokens_seen": 440795136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375727181544634, + "loss": 1.5547, + "theoretical_loss": 3.9688181309143657, + "tokens_seen": 440926208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375526579739218, + "loss": 1.4414, + "theoretical_loss": 3.9686883168708613, + "tokens_seen": 441057280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375325977933802, + "loss": 1.6541, + "theoretical_loss": 3.9685585521974556, + "tokens_seen": 441188352 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043751253761283856, + "loss": 1.68, + "theoretical_loss": 3.968428836860712, + "tokens_seen": 441319424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374924774322969, + "loss": 1.7028, + "theoretical_loss": 3.96829917082723, + "tokens_seen": 441450496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374724172517553, + "loss": 1.6575, + "theoretical_loss": 3.968169554063639, + "tokens_seen": 441581568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043745235707121365, + "loss": 1.6168, + "theoretical_loss": 3.968039986536601, + "tokens_seen": 441712640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043743229689067206, + "loss": 1.7393, + "theoretical_loss": 3.9679104682128106, + "tokens_seen": 441843712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374122367101304, + "loss": 1.7, + "theoretical_loss": 3.9677809990589967, + "tokens_seen": 441974784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373921765295888, + "loss": 1.4573, + "theoretical_loss": 3.9676515790419176, + "tokens_seen": 442105856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043737211634904715, + "loss": 1.5554, + "theoretical_loss": 3.967522208128366, + "tokens_seen": 442236928 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.43737781047821045, + "objective/train/avg_token_score": 0.016550948843359947, + "objective/train/docs_used": 262441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5659313201904297, + "objective/train/original_loss": 3.8397927284240723, + "objective/train/theoretical_loss": 3.9673928862851655, + "objective/train/tokens_used": 442368000, + "objective/train/unlikelihood_loss": 0.18158647418022156, + "theoretical_loss": 3.9673928862851655, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043735205616850557, + "loss": 1.5389, + "theoretical_loss": 3.9673928862851655, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043733199598796393, + "loss": 1.6709, + "theoretical_loss": 3.9672636134791737, + "tokens_seen": 442499072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373119358074223, + "loss": 1.6792, + "theoretical_loss": 3.9671343896772786, + "tokens_seen": 442630144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043729187562688065, + "loss": 1.5301, + "theoretical_loss": 3.967005214846401, + "tokens_seen": 442761216 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437271815446339, + "loss": 1.6247, + "theoretical_loss": 3.9668760889534944, + "tokens_seen": 442892288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043725175526579743, + "loss": 1.574, + "theoretical_loss": 3.966747011965544, + "tokens_seen": 443023360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372316950852558, + "loss": 1.6382, + "theoretical_loss": 3.9666179838495657, + "tokens_seen": 443154432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043721163490471416, + "loss": 1.5853, + "theoretical_loss": 3.96648900457261, + "tokens_seen": 443285504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371915747241725, + "loss": 1.5827, + "theoretical_loss": 3.9663600741017575, + "tokens_seen": 443416576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043717151454363093, + "loss": 1.5746, + "theoretical_loss": 3.9662311924041203, + "tokens_seen": 443547648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371514543630893, + "loss": 1.4653, + "theoretical_loss": 3.966102359446845, + "tokens_seen": 443678720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043713139418254766, + "loss": 1.5448, + "theoretical_loss": 3.9659735751971064, + "tokens_seen": 443809792 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437111334002006, + "loss": 1.6434, + "theoretical_loss": 3.9658448396221138, + "tokens_seen": 443940864 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.5477761626243591, + "objective/train/avg_token_score": 0.017558608204126358, + "objective/train/docs_used": 263839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9970813989639282, + "objective/train/original_loss": 3.930330753326416, + "objective/train/theoretical_loss": 3.9657804900774085, + "objective/train/tokens_used": 444006400, + "objective/train/unlikelihood_loss": 0.15241247415542603, + "theoretical_loss": 3.9657804900774085, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370912738214644, + "loss": 1.8145, + "theoretical_loss": 3.9657161526891076, + "tokens_seen": 444071936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370712136409228, + "loss": 1.6294, + "theoretical_loss": 3.9655875143653594, + "tokens_seen": 444203008 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043705115346038116, + "loss": 1.5046, + "theoretical_loss": 3.965458924618173, + "tokens_seen": 444334080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370310932798395, + "loss": 1.842, + "theoretical_loss": 3.9653303834148828, + "tokens_seen": 444465152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370110330992979, + "loss": 1.5623, + "theoretical_loss": 3.965201890722857, + "tokens_seen": 444596224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369909729187563, + "loss": 1.5008, + "theoretical_loss": 3.965073446509493, + "tokens_seen": 444727296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043697091273821467, + "loss": 1.5226, + "theoretical_loss": 3.9649450507422204, + "tokens_seen": 444858368 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043695085255767303, + "loss": 1.7281, + "theoretical_loss": 3.964816703388501, + "tokens_seen": 444989440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369307923771314, + "loss": 1.5668, + "theoretical_loss": 3.9646884044158277, + "tokens_seen": 445120512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043691073219658975, + "loss": 1.543, + "theoretical_loss": 3.9645601537917248, + "tokens_seen": 445251584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043689067201604817, + "loss": 1.6472, + "theoretical_loss": 3.9644319514837467, + "tokens_seen": 445382656 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043687061183550653, + "loss": 1.6752, + "theoretical_loss": 3.9643037974594817, + "tokens_seen": 445513728 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3345552384853363, + "objective/train/avg_token_score": 0.010209061205387115, + "objective/train/docs_used": 264540, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3382853269577026, + "objective/train/original_loss": 4.270390033721924, + "objective/train/theoretical_loss": 3.9641756916865463, + "objective/train/tokens_used": 445644800, + "objective/train/unlikelihood_loss": 0.15310260653495789, + "theoretical_loss": 3.9641756916865463, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368505516549649, + "loss": 1.6619, + "theoretical_loss": 3.9641756916865463, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043683049147442326, + "loss": 1.4821, + "theoretical_loss": 3.9640476341325916, + "tokens_seen": 445775872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043681043129388167, + "loss": 1.6354, + "theoretical_loss": 3.9639196247652966, + "tokens_seen": 445906944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043679037111334003, + "loss": 1.4849, + "theoretical_loss": 3.9637916635523736, + "tokens_seen": 446038016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367703109327984, + "loss": 1.4699, + "theoretical_loss": 3.9636637504615653, + "tokens_seen": 446169088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043675025075225676, + "loss": 1.4246, + "theoretical_loss": 3.9635358854606455, + "tokens_seen": 446300160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367301905717151, + "loss": 1.6212, + "theoretical_loss": 3.963408068517419, + "tokens_seen": 446431232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043671013039117354, + "loss": 1.7125, + "theoretical_loss": 3.963280299599722, + "tokens_seen": 446562304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366900702106319, + "loss": 1.4525, + "theoretical_loss": 3.9631525786754214, + "tokens_seen": 446693376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043667001003009026, + "loss": 1.5724, + "theoretical_loss": 3.9630249057124147, + "tokens_seen": 446824448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366499498495486, + "loss": 1.3806, + "theoretical_loss": 3.9628972806786305, + "tokens_seen": 446955520 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043662988966900704, + "loss": 1.7016, + "theoretical_loss": 3.9627697035420284, + "tokens_seen": 447086592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366098294884654, + "loss": 1.6344, + "theoretical_loss": 3.9626421742705986, + "tokens_seen": 447217664 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.34390273690223694, + "objective/train/avg_token_score": 0.016636604443192482, + "objective/train/docs_used": 265849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2081854343414307, + "objective/train/original_loss": 4.167159557342529, + "objective/train/theoretical_loss": 3.9625784275743294, + "objective/train/tokens_used": 447283200, + "objective/train/unlikelihood_loss": 0.16087184846401215, + "theoretical_loss": 3.9625784275743294, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043658976930792376, + "loss": 1.4704, + "theoretical_loss": 3.9625146928323627, + "tokens_seen": 447348736 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656970912738213, + "loss": 1.3915, + "theoretical_loss": 3.9623872591953715, + "tokens_seen": 447479808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365496489468405, + "loss": 1.6532, + "theoretical_loss": 3.9622598733277083, + "tokens_seen": 447610880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365295887662989, + "loss": 1.5888, + "theoretical_loss": 3.962132535197486, + "tokens_seen": 447741952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043650952858575727, + "loss": 1.3799, + "theoretical_loss": 3.9620052447728478, + "tokens_seen": 447873024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648946840521563, + "loss": 1.5288, + "theoretical_loss": 3.9618780020219693, + "tokens_seen": 448004096 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436469408224674, + "loss": 1.5376, + "theoretical_loss": 3.961750806913054, + "tokens_seen": 448135168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364493480441324, + "loss": 1.5963, + "theoretical_loss": 3.961623659414338, + "tokens_seen": 448266240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043642928786359077, + "loss": 1.5641, + "theoretical_loss": 3.961496559494087, + "tokens_seen": 448397312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043640922768304913, + "loss": 1.3678, + "theoretical_loss": 3.9613695071205974, + "tokens_seen": 448528384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363891675025075, + "loss": 1.4971, + "theoretical_loss": 3.9612425022621953, + "tokens_seen": 448659456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363691073219659, + "loss": 1.5659, + "theoretical_loss": 3.961115544887238, + "tokens_seen": 448790528 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3955278694629669, + "objective/train/avg_token_score": 0.013740437105298042, + "objective/train/docs_used": 266141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3886208534240723, + "objective/train/original_loss": 3.7399308681488037, + "objective/train/theoretical_loss": 3.960988634964113, + "objective/train/tokens_used": 448921600, + "objective/train/unlikelihood_loss": 0.21553213894367218, + "theoretical_loss": 3.960988634964113, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363490471414243, + "loss": 1.6211, + "theoretical_loss": 3.960988634964113, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632898696088264, + "loss": 1.5591, + "theoretical_loss": 3.9608617724612367, + "tokens_seen": 449052672 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436308926780341, + "loss": 1.3739, + "theoretical_loss": 3.960734957347058, + "tokens_seen": 449183744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043628886659979936, + "loss": 1.6942, + "theoretical_loss": 3.9606081895900545, + "tokens_seen": 449314816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362688064192578, + "loss": 1.6639, + "theoretical_loss": 3.9604814691587347, + "tokens_seen": 449445888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043624874623871614, + "loss": 1.4516, + "theoretical_loss": 3.960354796021636, + "tokens_seen": 449576960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362286860581745, + "loss": 1.4179, + "theoretical_loss": 3.960228170147327, + "tokens_seen": 449708032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043620862587763286, + "loss": 1.3733, + "theoretical_loss": 3.960101591504406, + "tokens_seen": 449839104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361885656970913, + "loss": 1.5806, + "theoretical_loss": 3.9599750600615025, + "tokens_seen": 449970176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361685055165497, + "loss": 1.3499, + "theoretical_loss": 3.959848575787273, + "tokens_seen": 450101248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043614844533600806, + "loss": 1.55, + "theoretical_loss": 3.9597221386504073, + "tokens_seen": 450232320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361283851554664, + "loss": 1.5825, + "theoretical_loss": 3.9595957486196225, + "tokens_seen": 450363392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361083249749248, + "loss": 1.4968, + "theoretical_loss": 3.9594694056636675, + "tokens_seen": 450494464 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3633919954299927, + "objective/train/avg_token_score": 0.05124437436461449, + "objective/train/docs_used": 266764, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3248687982559204, + "objective/train/original_loss": 4.117973327636719, + "objective/train/theoretical_loss": 3.9594062518289923, + "objective/train/tokens_used": 450560000, + "objective/train/unlikelihood_loss": 0.1388237476348877, + "theoretical_loss": 3.9594062518289923, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360882647943832, + "loss": 1.3466, + "theoretical_loss": 3.9593431097513188, + "tokens_seen": 450625536 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043606820461384156, + "loss": 1.5877, + "theoretical_loss": 3.9592168608513854, + "tokens_seen": 450756608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360481444332999, + "loss": 1.4062, + "theoretical_loss": 3.9590906589327037, + "tokens_seen": 450887680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360280842527583, + "loss": 1.3673, + "theoretical_loss": 3.958964503964141, + "tokens_seen": 451018752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360080240722167, + "loss": 1.4466, + "theoretical_loss": 3.9588383959145945, + "tokens_seen": 451149824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043598796389167507, + "loss": 1.4174, + "theoretical_loss": 3.95871233475299, + "tokens_seen": 451280896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043596790371113343, + "loss": 1.5887, + "theoretical_loss": 3.9585863204482834, + "tokens_seen": 451411968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359478435305918, + "loss": 1.5364, + "theoretical_loss": 3.9584603529694604, + "tokens_seen": 451543040 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043592778335005015, + "loss": 1.6187, + "theoretical_loss": 3.958334432285537, + "tokens_seen": 451674112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043590772316950857, + "loss": 1.5349, + "theoretical_loss": 3.9582085583655564, + "tokens_seen": 451805184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043588766298896693, + "loss": 1.3348, + "theoretical_loss": 3.9580827311785924, + "tokens_seen": 451936256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358676028084253, + "loss": 1.4182, + "theoretical_loss": 3.9579569506937498, + "tokens_seen": 452067328 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.41373410820961, + "objective/train/avg_token_score": 0.03839399293065071, + "objective/train/docs_used": 268192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.432794213294983, + "objective/train/original_loss": 4.107253551483154, + "objective/train/theoretical_loss": 3.9578312168801597, + "objective/train/tokens_used": 452198400, + "objective/train/unlikelihood_loss": 0.13305017352104187, + "theoretical_loss": 3.9578312168801597, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043584754262788366, + "loss": 1.4714, + "theoretical_loss": 3.9578312168801597, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043582748244734207, + "loss": 1.683, + "theoretical_loss": 3.9577055297069856, + "tokens_seen": 452329472 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043580742226680043, + "loss": 1.4859, + "theoretical_loss": 3.9575798891434184, + "tokens_seen": 452460544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357873620862588, + "loss": 1.4563, + "theoretical_loss": 3.957454295158679, + "tokens_seen": 452591616 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043576730190571716, + "loss": 1.5263, + "theoretical_loss": 3.9573287477220167, + "tokens_seen": 452722688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357472417251755, + "loss": 1.6256, + "theoretical_loss": 3.957203246802711, + "tokens_seen": 452853760 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043572718154463394, + "loss": 1.6589, + "theoretical_loss": 3.9570777923700704, + "tokens_seen": 452984832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357071213640923, + "loss": 1.3931, + "theoretical_loss": 3.956952384393432, + "tokens_seen": 453115904 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043568706118355066, + "loss": 1.5144, + "theoretical_loss": 3.9568270228421616, + "tokens_seen": 453246976 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435667001003009, + "loss": 1.4202, + "theoretical_loss": 3.9567017076856565, + "tokens_seen": 453378048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043564694082246744, + "loss": 1.5056, + "theoretical_loss": 3.9565764388933395, + "tokens_seen": 453509120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356268806419258, + "loss": 1.5587, + "theoretical_loss": 3.9564512164346652, + "tokens_seen": 453640192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043560682046138417, + "loss": 1.6948, + "theoretical_loss": 3.956326040279116, + "tokens_seen": 453771264 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4516128897666931, + "objective/train/avg_token_score": 0.02117766998708248, + "objective/train/docs_used": 268717, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.552639126777649, + "objective/train/original_loss": 4.055312156677246, + "objective/train/theoretical_loss": 3.956263469555484, + "objective/train/tokens_used": 453836800, + "objective/train/unlikelihood_loss": 0.12762990593910217, + "theoretical_loss": 3.956263469555484, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043558676028084253, + "loss": 1.6062, + "theoretical_loss": 3.956200910396202, + "tokens_seen": 453902336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355667001003009, + "loss": 1.3842, + "theoretical_loss": 3.956075826755465, + "tokens_seen": 454033408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355466399197593, + "loss": 1.5806, + "theoretical_loss": 3.955950789326474, + "tokens_seen": 454164480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043552657973921767, + "loss": 1.4914, + "theoretical_loss": 3.955825798078826, + "tokens_seen": 454295552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043550651955867603, + "loss": 1.4215, + "theoretical_loss": 3.9557008529821474, + "tokens_seen": 454426624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354864593781344, + "loss": 1.3912, + "theoretical_loss": 3.955575954006095, + "tokens_seen": 454557696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354663991975928, + "loss": 1.5038, + "theoretical_loss": 3.955451101120352, + "tokens_seen": 454688768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043544633901705117, + "loss": 1.6058, + "theoretical_loss": 3.9553262942946312, + "tokens_seen": 454819840 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043542627883650953, + "loss": 1.4004, + "theoretical_loss": 3.9552015334986743, + "tokens_seen": 454950912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354062186559679, + "loss": 1.6557, + "theoretical_loss": 3.95507681870225, + "tokens_seen": 455081984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353861584754263, + "loss": 1.3329, + "theoretical_loss": 3.954952149875159, + "tokens_seen": 455213056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353660982948847, + "loss": 1.506, + "theoretical_loss": 3.954827526987226, + "tokens_seen": 455344128 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.35752686858177185, + "objective/train/avg_token_score": 0.016128363087773323, + "objective/train/docs_used": 269967, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1892235279083252, + "objective/train/original_loss": 3.846442937850952, + "objective/train/theoretical_loss": 3.954702950008308, + "objective/train/tokens_used": 455475200, + "objective/train/unlikelihood_loss": 0.21875594556331635, + "theoretical_loss": 3.954702950008308, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043534603811434304, + "loss": 1.2484, + "theoretical_loss": 3.954702950008308, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353259779338014, + "loss": 1.3839, + "theoretical_loss": 3.9545784189082887, + "tokens_seen": 455606272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043530591775325976, + "loss": 1.5678, + "theoretical_loss": 3.9544539336570796, + "tokens_seen": 455737344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352858575727182, + "loss": 1.4671, + "theoretical_loss": 3.954329494224623, + "tokens_seen": 455868416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043526579739217654, + "loss": 1.5832, + "theoretical_loss": 3.9542051005808867, + "tokens_seen": 455999488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352457372116349, + "loss": 1.5173, + "theoretical_loss": 3.954080752695868, + "tokens_seen": 456130560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043522567703109326, + "loss": 1.4201, + "theoretical_loss": 3.953956450539594, + "tokens_seen": 456261632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352056168505517, + "loss": 1.634, + "theoretical_loss": 3.953832194082117, + "tokens_seen": 456392704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043518555667001004, + "loss": 1.5989, + "theoretical_loss": 3.95370798329352, + "tokens_seen": 456523776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351654964894684, + "loss": 1.5042, + "theoretical_loss": 3.9535838181439127, + "tokens_seen": 456654848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043514543630892677, + "loss": 1.5895, + "theoretical_loss": 3.953459698603434, + "tokens_seen": 456785920 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043512537612838513, + "loss": 1.611, + "theoretical_loss": 3.95333562464225, + "tokens_seen": 456916992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043510531594784355, + "loss": 1.5907, + "theoretical_loss": 3.953211596230556, + "tokens_seen": 457048064 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.5039711594581604, + "objective/train/avg_token_score": 0.018044468015432358, + "objective/train/docs_used": 270622, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7069590091705322, + "objective/train/original_loss": 3.6348330974578857, + "objective/train/theoretical_loss": 3.953149599096461, + "objective/train/tokens_used": 457113600, + "objective/train/unlikelihood_loss": 0.2166004627943039, + "theoretical_loss": 3.953149599096461, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350852557673019, + "loss": 1.5883, + "theoretical_loss": 3.9530876133385737, + "tokens_seen": 457179136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043506519558676027, + "loss": 1.4272, + "theoretical_loss": 3.9529636759365543, + "tokens_seen": 457310208 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043504513540621863, + "loss": 1.448, + "theoretical_loss": 3.952839783994776, + "tokens_seen": 457441280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043502507522567705, + "loss": 1.4977, + "theoretical_loss": 3.9527159374835463, + "tokens_seen": 457572352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350050150451354, + "loss": 1.4147, + "theoretical_loss": 3.9525921363731977, + "tokens_seen": 457703424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349849548645938, + "loss": 1.6583, + "theoretical_loss": 3.9524683806340937, + "tokens_seen": 457834496 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043496489468405214, + "loss": 1.6028, + "theoretical_loss": 3.9523446702366245, + "tokens_seen": 457965568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349448345035105, + "loss": 1.5376, + "theoretical_loss": 3.9522210051512077, + "tokens_seen": 458096640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349247743229689, + "loss": 1.4196, + "theoretical_loss": 3.9520973853482886, + "tokens_seen": 458227712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349047141424273, + "loss": 1.5257, + "theoretical_loss": 3.951973810798341, + "tokens_seen": 458358784 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043488465396188564, + "loss": 1.5334, + "theoretical_loss": 3.9518502814718652, + "tokens_seen": 458489856 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434864593781344, + "loss": 1.3228, + "theoretical_loss": 3.951726797339391, + "tokens_seen": 458620928 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.39320626854896545, + "objective/train/avg_token_score": 0.018131617456674576, + "objective/train/docs_used": 271952, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.354129433631897, + "objective/train/original_loss": 3.985093355178833, + "objective/train/theoretical_loss": 3.9516033583714734, + "objective/train/tokens_used": 458752000, + "objective/train/unlikelihood_loss": 0.19177991151809692, + "theoretical_loss": 3.9516033583714734, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348445336008024, + "loss": 1.514, + "theoretical_loss": 3.9516033583714734, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348244734202608, + "loss": 1.4862, + "theoretical_loss": 3.9514799645386973, + "tokens_seen": 458883072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480441323971914, + "loss": 1.5574, + "theoretical_loss": 3.951356615811674, + "tokens_seen": 459014144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347843530591775, + "loss": 1.3874, + "theoretical_loss": 3.951233312161042, + "tokens_seen": 459145216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476429287863587, + "loss": 1.5508, + "theoretical_loss": 3.9511100535574686, + "tokens_seen": 459276288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347442326980943, + "loss": 1.5419, + "theoretical_loss": 3.9509868399716472, + "tokens_seen": 459407360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472417251755265, + "loss": 1.2042, + "theoretical_loss": 3.950863671374299, + "tokens_seen": 459538432 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434704112337011, + "loss": 1.5611, + "theoretical_loss": 3.9507405477361726, + "tokens_seen": 459669504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346840521564694, + "loss": 1.4663, + "theoretical_loss": 3.9506174690280442, + "tokens_seen": 459800576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043466399197592784, + "loss": 1.5522, + "theoretical_loss": 3.9504944352207176, + "tokens_seen": 459931648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346439317953862, + "loss": 1.5421, + "theoretical_loss": 3.9503714462850237, + "tokens_seen": 460062720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043462387161484457, + "loss": 1.557, + "theoretical_loss": 3.950248502191819, + "tokens_seen": 460193792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043460381143430293, + "loss": 1.5097, + "theoretical_loss": 3.9501256029119896, + "tokens_seen": 460324864 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4094574749469757, + "objective/train/avg_token_score": 0.011406373232603073, + "objective/train/docs_used": 272724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6004735231399536, + "objective/train/original_loss": 4.182957649230957, + "objective/train/theoretical_loss": 3.950064170068, + "objective/train/tokens_used": 460390400, + "objective/train/unlikelihood_loss": 0.14253197610378265, + "theoretical_loss": 3.950064170068, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345837512537613, + "loss": 1.5761, + "theoretical_loss": 3.9500027484164484, + "tokens_seen": 460455936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345636910732197, + "loss": 1.3552, + "theoretical_loss": 3.9498799386761334, + "tokens_seen": 460587008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043454363089267807, + "loss": 1.4481, + "theoretical_loss": 3.9497571736620127, + "tokens_seen": 460718080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043452357071213643, + "loss": 1.4637, + "theoretical_loss": 3.9496344533450793, + "tokens_seen": 460849152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345035105315948, + "loss": 1.6593, + "theoretical_loss": 3.9495117776963538, + "tokens_seen": 460980224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344834503510532, + "loss": 1.495, + "theoretical_loss": 3.9493891466868845, + "tokens_seen": 461111296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043446339017051157, + "loss": 1.4484, + "theoretical_loss": 3.9492665602877457, + "tokens_seen": 461242368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043444332998996993, + "loss": 1.4955, + "theoretical_loss": 3.9491440184700393, + "tokens_seen": 461373440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344232698094283, + "loss": 1.4501, + "theoretical_loss": 3.9490215212048936, + "tokens_seen": 461504512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344032096288867, + "loss": 1.4812, + "theoretical_loss": 3.948899068463465, + "tokens_seen": 461635584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343831494483451, + "loss": 1.6095, + "theoretical_loss": 3.948776660216935, + "tokens_seen": 461766656 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043436308926780344, + "loss": 1.4606, + "theoretical_loss": 3.948654296436513, + "tokens_seen": 461897728 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4204545319080353, + "objective/train/avg_token_score": 0.009838702157139778, + "objective/train/docs_used": 273842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5137208700180054, + "objective/train/original_loss": 3.8193166255950928, + "objective/train/theoretical_loss": 3.9485319770934355, + "objective/train/tokens_used": 462028800, + "objective/train/unlikelihood_loss": 0.21145619451999664, + "theoretical_loss": 3.9485319770934355, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343430290872618, + "loss": 1.5253, + "theoretical_loss": 3.9485319770934355, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043432296890672016, + "loss": 1.4442, + "theoretical_loss": 3.9484097021589646, + "tokens_seen": 462159872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343029087261786, + "loss": 1.3611, + "theoretical_loss": 3.94828747160439, + "tokens_seen": 462290944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428284854563694, + "loss": 1.5169, + "theoretical_loss": 3.9481652854010285, + "tokens_seen": 462422016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342627883650953, + "loss": 1.602, + "theoretical_loss": 3.9480431435202226, + "tokens_seen": 462553088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043424272818455367, + "loss": 1.4654, + "theoretical_loss": 3.9479210459333407, + "tokens_seen": 462684160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342226680040121, + "loss": 1.4747, + "theoretical_loss": 3.94779899261178, + "tokens_seen": 462815232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043420260782347044, + "loss": 1.6772, + "theoretical_loss": 3.9476769835269634, + "tokens_seen": 462946304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341825476429288, + "loss": 1.6903, + "theoretical_loss": 3.94755501865034, + "tokens_seen": 463077376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043416248746238717, + "loss": 1.5698, + "theoretical_loss": 3.9474330979533843, + "tokens_seen": 463208448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043414242728184553, + "loss": 1.5586, + "theoretical_loss": 3.947311221407599, + "tokens_seen": 463339520 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043412236710130395, + "loss": 1.5921, + "theoretical_loss": 3.9471893889845138, + "tokens_seen": 463470592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341023069207623, + "loss": 1.3821, + "theoretical_loss": 3.947067600655682, + "tokens_seen": 463601664 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3299120366573334, + "objective/train/avg_token_score": 0.027434295043349266, + "objective/train/docs_used": 274430, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1526551246643066, + "objective/train/original_loss": 3.9500741958618164, + "objective/train/theoretical_loss": 3.94700672301773, + "objective/train/tokens_used": 463667200, + "objective/train/unlikelihood_loss": 0.1668168306350708, + "theoretical_loss": 3.94700672301773, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043408224674022067, + "loss": 1.4242, + "theoretical_loss": 3.946945856392687, + "tokens_seen": 463732736 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043406218655967903, + "loss": 1.3769, + "theoretical_loss": 3.9468241561671342, + "tokens_seen": 463863808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043404212637913745, + "loss": 1.3464, + "theoretical_loss": 3.946702499950659, + "tokens_seen": 463994880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340220661985958, + "loss": 1.5064, + "theoretical_loss": 3.946580887714921, + "tokens_seen": 464125952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340020060180542, + "loss": 1.322, + "theoretical_loss": 3.946459319431608, + "tokens_seen": 464257024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043398194583751254, + "loss": 1.5324, + "theoretical_loss": 3.9463377950724317, + "tokens_seen": 464388096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339618856569709, + "loss": 1.3533, + "theoretical_loss": 3.9462163146091305, + "tokens_seen": 464519168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339418254764293, + "loss": 1.5288, + "theoretical_loss": 3.9460948780134713, + "tokens_seen": 464650240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339217652958877, + "loss": 1.5073, + "theoretical_loss": 3.9459734852572437, + "tokens_seen": 464781312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043390170511534604, + "loss": 1.5032, + "theoretical_loss": 3.945852136312265, + "tokens_seen": 464912384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338816449348044, + "loss": 1.6266, + "theoretical_loss": 3.94573083115038, + "tokens_seen": 465043456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338615847542628, + "loss": 1.5327, + "theoretical_loss": 3.9456095697434566, + "tokens_seen": 465174528 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.32600194215774536, + "objective/train/avg_token_score": 0.02726374939084053, + "objective/train/docs_used": 275809, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0033814907073975, + "objective/train/original_loss": 3.863251209259033, + "objective/train/theoretical_loss": 3.945488352063391, + "objective/train/tokens_used": 465305600, + "objective/train/unlikelihood_loss": 0.22456102073192596, + "theoretical_loss": 3.945488352063391, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338415245737212, + "loss": 1.4332, + "theoretical_loss": 3.945488352063391, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043382146439317954, + "loss": 1.4683, + "theoretical_loss": 3.945367178082104, + "tokens_seen": 465436672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338014042126379, + "loss": 1.4562, + "theoretical_loss": 3.945246047771543, + "tokens_seen": 465567744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043378134403209627, + "loss": 1.6305, + "theoretical_loss": 3.945124961103682, + "tokens_seen": 465698816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337612838515547, + "loss": 1.5113, + "theoretical_loss": 3.945003918050518, + "tokens_seen": 465829888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043374122367101305, + "loss": 1.5304, + "theoretical_loss": 3.9448829185840784, + "tokens_seen": 465960960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337211634904714, + "loss": 1.4951, + "theoretical_loss": 3.944761962676412, + "tokens_seen": 466092032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043370110330992977, + "loss": 1.4054, + "theoretical_loss": 3.9446410502995963, + "tokens_seen": 466223104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336810431293882, + "loss": 1.4125, + "theoretical_loss": 3.9445201814257334, + "tokens_seen": 466354176 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043366098294884655, + "loss": 1.4092, + "theoretical_loss": 3.9443993560269504, + "tokens_seen": 466485248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336409227683049, + "loss": 1.3047, + "theoretical_loss": 3.944278574075402, + "tokens_seen": 466616320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336208625877633, + "loss": 1.5973, + "theoretical_loss": 3.9441578355432663, + "tokens_seen": 466747392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043360080240722164, + "loss": 1.6775, + "theoretical_loss": 3.9440371404027488, + "tokens_seen": 466878464 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4305962920188904, + "objective/train/avg_token_score": 0.008782853372395039, + "objective/train/docs_used": 276320, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.639618158340454, + "objective/train/original_loss": 4.319045543670654, + "objective/train/theoretical_loss": 3.943976809095668, + "objective/train/tokens_used": 466944000, + "objective/train/unlikelihood_loss": 0.18612205982208252, + "theoretical_loss": 3.943976809095668, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043358074222668005, + "loss": 1.5497, + "theoretical_loss": 3.94391648862608, + "tokens_seen": 467009536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335606820461384, + "loss": 1.5624, + "theoretical_loss": 3.9437958801855157, + "tokens_seen": 467140608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335406218655968, + "loss": 1.5378, + "theoretical_loss": 3.9436753150533375, + "tokens_seen": 467271680 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043352056168505514, + "loss": 1.7575, + "theoretical_loss": 3.9435547932018524, + "tokens_seen": 467402752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043350050150451356, + "loss": 1.7, + "theoretical_loss": 3.943434314603394, + "tokens_seen": 467533824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334804413239719, + "loss": 1.4659, + "theoretical_loss": 3.9433138792303177, + "tokens_seen": 467664896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334603811434303, + "loss": 1.3896, + "theoretical_loss": 3.943193487055009, + "tokens_seen": 467795968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043344032096288864, + "loss": 1.3853, + "theoretical_loss": 3.9430731380498765, + "tokens_seen": 467927040 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433420260782347, + "loss": 1.3526, + "theoretical_loss": 3.9429528321873537, + "tokens_seen": 468058112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334002006018054, + "loss": 1.5852, + "theoretical_loss": 3.9428325694398993, + "tokens_seen": 468189184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333801404212638, + "loss": 1.6151, + "theoretical_loss": 3.94271234978, + "tokens_seen": 468320256 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043336008024072215, + "loss": 1.5538, + "theoretical_loss": 3.942592173180163, + "tokens_seen": 468451328 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.5079423189163208, + "objective/train/avg_token_score": 0.011020613834261894, + "objective/train/docs_used": 277290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8621379137039185, + "objective/train/original_loss": 3.9069674015045166, + "objective/train/theoretical_loss": 3.942472039612926, + "objective/train/tokens_used": 468582400, + "objective/train/unlikelihood_loss": 0.13427872955799103, + "theoretical_loss": 3.942472039612926, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333400200601805, + "loss": 1.5221, + "theoretical_loss": 3.942472039612926, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333199598796389, + "loss": 1.543, + "theoretical_loss": 3.9423519490508476, + "tokens_seen": 468713472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332998996990973, + "loss": 1.5703, + "theoretical_loss": 3.9422319014665144, + "tokens_seen": 468844544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327983951855565, + "loss": 1.419, + "theoretical_loss": 3.942111896832537, + "tokens_seen": 468975616 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433259779338014, + "loss": 1.4121, + "theoretical_loss": 3.9419919351215498, + "tokens_seen": 469106688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323971915747243, + "loss": 1.481, + "theoretical_loss": 3.941872016306215, + "tokens_seen": 469237760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332196589769308, + "loss": 1.5854, + "theoretical_loss": 3.9417521403592186, + "tokens_seen": 469368832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043319959879638915, + "loss": 1.7019, + "theoretical_loss": 3.94163230725327, + "tokens_seen": 469499904 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043317953861584757, + "loss": 1.5685, + "theoretical_loss": 3.941512516961107, + "tokens_seen": 469630976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043315947843530593, + "loss": 1.4466, + "theoretical_loss": 3.9413927694554887, + "tokens_seen": 469762048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043313941825476435, + "loss": 1.5199, + "theoretical_loss": 3.9412730647092022, + "tokens_seen": 469893120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331193580742227, + "loss": 1.4827, + "theoretical_loss": 3.9411534026950568, + "tokens_seen": 470024192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043309929789368107, + "loss": 1.6589, + "theoretical_loss": 3.9410337833858886, + "tokens_seen": 470155264 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3777492642402649, + "objective/train/avg_token_score": 0.027359778061509132, + "objective/train/docs_used": 277851, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.291692852973938, + "objective/train/original_loss": 4.366660118103027, + "objective/train/theoretical_loss": 3.9409739897371896, + "objective/train/tokens_used": 470220800, + "objective/train/unlikelihood_loss": 0.15265023708343506, + "theoretical_loss": 3.9409739897371896, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043307923771313943, + "loss": 1.3202, + "theoretical_loss": 3.9409142067545586, + "tokens_seen": 470286336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043305917753259785, + "loss": 1.2957, + "theoretical_loss": 3.940794672773951, + "tokens_seen": 470417408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330391173520562, + "loss": 1.5722, + "theoretical_loss": 3.9406751814169763, + "tokens_seen": 470548480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330190571715146, + "loss": 1.4456, + "theoretical_loss": 3.940555732656569, + "tokens_seen": 470679552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043299899699097294, + "loss": 1.4136, + "theoretical_loss": 3.9404363264656883, + "tokens_seen": 470810624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329789368104313, + "loss": 1.3883, + "theoretical_loss": 3.9403169628173185, + "tokens_seen": 470941696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329588766298897, + "loss": 1.5931, + "theoretical_loss": 3.9401976416844677, + "tokens_seen": 471072768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329388164493481, + "loss": 1.5737, + "theoretical_loss": 3.94007836304017, + "tokens_seen": 471203840 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043291875626880644, + "loss": 1.6797, + "theoretical_loss": 3.9399591268574836, + "tokens_seen": 471334912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328986960882648, + "loss": 1.5247, + "theoretical_loss": 3.93983993310949, + "tokens_seen": 471465984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328786359077232, + "loss": 1.6422, + "theoretical_loss": 3.9397207817692967, + "tokens_seen": 471597056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328585757271816, + "loss": 1.6177, + "theoretical_loss": 3.9396016728100363, + "tokens_seen": 471728128 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.33987048268318176, + "objective/train/avg_token_score": 0.018589701503515244, + "objective/train/docs_used": 279270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2118350267410278, + "objective/train/original_loss": 3.926861524581909, + "objective/train/theoretical_loss": 3.939482606204863, + "objective/train/tokens_used": 471859200, + "objective/train/unlikelihood_loss": 0.1467958688735962, + "theoretical_loss": 3.939482606204863, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043283851554663994, + "loss": 1.5283, + "theoretical_loss": 3.939482606204863, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328184553660983, + "loss": 1.4258, + "theoretical_loss": 3.939363581926959, + "tokens_seen": 471990272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043279839518555667, + "loss": 1.4664, + "theoretical_loss": 3.939244599949528, + "tokens_seen": 472121344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327783350050151, + "loss": 1.3412, + "theoretical_loss": 3.9391256602457996, + "tokens_seen": 472252416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043275827482447345, + "loss": 1.4782, + "theoretical_loss": 3.9390067627890275, + "tokens_seen": 472383488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327382146439318, + "loss": 1.4871, + "theoretical_loss": 3.9388879075524907, + "tokens_seen": 472514560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043271815446339017, + "loss": 1.6205, + "theoretical_loss": 3.9387690945094906, + "tokens_seen": 472645632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326980942828486, + "loss": 1.3513, + "theoretical_loss": 3.938650323633354, + "tokens_seen": 472776704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043267803410230695, + "loss": 1.6774, + "theoretical_loss": 3.9385315948974315, + "tokens_seen": 472907776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326579739217653, + "loss": 1.542, + "theoretical_loss": 3.938412908275099, + "tokens_seen": 473038848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326379137412237, + "loss": 1.4591, + "theoretical_loss": 3.938294263739755, + "tokens_seen": 473169920 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043261785356068204, + "loss": 1.5059, + "theoretical_loss": 3.938175661264823, + "tokens_seen": 473300992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043259779338014045, + "loss": 1.4335, + "theoretical_loss": 3.938057100823751, + "tokens_seen": 473432064 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4881476163864136, + "objective/train/avg_token_score": 0.017582368105649948, + "objective/train/docs_used": 279967, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6766897439956665, + "objective/train/original_loss": 3.8044707775115967, + "objective/train/theoretical_loss": 3.9379978363576216, + "objective/train/tokens_used": 473497600, + "objective/train/unlikelihood_loss": 0.20410603284835815, + "theoretical_loss": 3.9379978363576216, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325777331995988, + "loss": 1.7232, + "theoretical_loss": 3.937938582390011, + "tokens_seen": 473563136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325576730190572, + "loss": 1.5215, + "theoretical_loss": 3.9378201059370976, + "tokens_seen": 473694208 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043253761283851554, + "loss": 1.4315, + "theoretical_loss": 3.9377016714385316, + "tokens_seen": 473825280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043251755265797396, + "loss": 1.5092, + "theoretical_loss": 3.937583278867857, + "tokens_seen": 473956352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324974924774323, + "loss": 1.5942, + "theoretical_loss": 3.9374649281986405, + "tokens_seen": 474087424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324774322968907, + "loss": 1.5666, + "theoretical_loss": 3.937346619404475, + "tokens_seen": 474218496 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043245737211634904, + "loss": 1.666, + "theoretical_loss": 3.937228352458976, + "tokens_seen": 474349568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324373119358074, + "loss": 1.6075, + "theoretical_loss": 3.9371101273357825, + "tokens_seen": 474480640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324172517552658, + "loss": 1.4956, + "theoretical_loss": 3.9369919440085592, + "tokens_seen": 474611712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323971915747242, + "loss": 1.4977, + "theoretical_loss": 3.9368738024509926, + "tokens_seen": 474742784 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043237713139418255, + "loss": 1.6371, + "theoretical_loss": 3.936755702636794, + "tokens_seen": 474873856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323570712136409, + "loss": 1.6294, + "theoretical_loss": 3.936637644539699, + "tokens_seen": 475004928 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.469696968793869, + "objective/train/avg_token_score": 0.015840815380215645, + "objective/train/docs_used": 280356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.593431830406189, + "objective/train/original_loss": 3.748237133026123, + "objective/train/theoretical_loss": 3.936519628133466, + "objective/train/tokens_used": 475136000, + "objective/train/unlikelihood_loss": 0.19854024052619934, + "theoretical_loss": 3.936519628133466, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323370110330993, + "loss": 1.6438, + "theoretical_loss": 3.936519628133466, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323169508525577, + "loss": 1.4211, + "theoretical_loss": 3.9364016533918766, + "tokens_seen": 475267072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043229689067201605, + "loss": 1.4308, + "theoretical_loss": 3.9362837202887384, + "tokens_seen": 475398144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322768304914744, + "loss": 1.5334, + "theoretical_loss": 3.9361658287978813, + "tokens_seen": 475529216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043225677031093283, + "loss": 1.4563, + "theoretical_loss": 3.936047978893158, + "tokens_seen": 475660288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322367101303912, + "loss": 1.3508, + "theoretical_loss": 3.9359301705484455, + "tokens_seen": 475791360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043221664994984955, + "loss": 1.6533, + "theoretical_loss": 3.935812403737646, + "tokens_seen": 475922432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321965897693079, + "loss": 1.7426, + "theoretical_loss": 3.9356946784346825, + "tokens_seen": 476053504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321765295887663, + "loss": 1.5579, + "theoretical_loss": 3.9355769946135037, + "tokens_seen": 476184576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321564694082247, + "loss": 1.5945, + "theoretical_loss": 3.9354593522480803, + "tokens_seen": 476315648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043213640922768306, + "loss": 1.5677, + "theoretical_loss": 3.935341751312408, + "tokens_seen": 476446720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321163490471414, + "loss": 1.4719, + "theoretical_loss": 3.935224191780505, + "tokens_seen": 476577792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320962888665998, + "loss": 1.7156, + "theoretical_loss": 3.9351066736264126, + "tokens_seen": 476708864 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4970674514770508, + "objective/train/avg_token_score": 0.05302438512444496, + "objective/train/docs_used": 281398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6894476413726807, + "objective/train/original_loss": 4.0028076171875, + "objective/train/theoretical_loss": 3.93504793005794, + "objective/train/tokens_used": 476774400, + "objective/train/unlikelihood_loss": 0.18129520118236542, + "theoretical_loss": 3.93504793005794, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320762286860582, + "loss": 1.544, + "theoretical_loss": 3.9349891968241963, + "tokens_seen": 476839936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043205616850551656, + "loss": 1.6091, + "theoretical_loss": 3.934871761347945, + "tokens_seen": 476971008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320361083249749, + "loss": 1.3174, + "theoretical_loss": 3.93475436717177, + "tokens_seen": 477102080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320160481444333, + "loss": 1.4658, + "theoretical_loss": 3.934637014269808, + "tokens_seen": 477233152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043199598796389165, + "loss": 1.2387, + "theoretical_loss": 3.9345197026162158, + "tokens_seen": 477364224 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043197592778335006, + "loss": 1.3269, + "theoretical_loss": 3.934402432185176, + "tokens_seen": 477495296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319558676028084, + "loss": 1.6676, + "theoretical_loss": 3.9342852029508935, + "tokens_seen": 477626368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319358074222668, + "loss": 1.4119, + "theoretical_loss": 3.934168014887597, + "tokens_seen": 477757440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043191574724172515, + "loss": 1.464, + "theoretical_loss": 3.934050867969537, + "tokens_seen": 477888512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043189568706118357, + "loss": 1.5136, + "theoretical_loss": 3.93393376217099, + "tokens_seen": 478019584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043187562688064193, + "loss": 1.5275, + "theoretical_loss": 3.933816697466252, + "tokens_seen": 478150656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318555667001003, + "loss": 1.6569, + "theoretical_loss": 3.9336996738296444, + "tokens_seen": 478281728 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4156891405582428, + "objective/train/avg_token_score": 0.021102596074342728, + "objective/train/docs_used": 281924, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5788319110870361, + "objective/train/original_loss": 4.477541446685791, + "objective/train/theoretical_loss": 3.9335826912355114, + "objective/train/tokens_used": 478412800, + "objective/train/unlikelihood_loss": 0.12658530473709106, + "theoretical_loss": 3.9335826912355114, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183550651955865, + "loss": 1.3921, + "theoretical_loss": 3.9335826912355114, + "tokens_seen": 478412800 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431815446339017, + "loss": 1.3283, + "theoretical_loss": 3.9334657496582195, + "tokens_seen": 478543872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043179538615847543, + "loss": 1.3853, + "theoretical_loss": 3.9333488490721598, + "tokens_seen": 478674944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317753259779338, + "loss": 1.4112, + "theoretical_loss": 3.933231989451744, + "tokens_seen": 478806016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043175526579739215, + "loss": 1.4039, + "theoretical_loss": 3.933115170771409, + "tokens_seen": 478937088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317352056168505, + "loss": 1.37, + "theoretical_loss": 3.932998393005613, + "tokens_seen": 479068160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043171514543630893, + "loss": 1.3066, + "theoretical_loss": 3.9328816561288384, + "tokens_seen": 479199232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316950852557673, + "loss": 1.4299, + "theoretical_loss": 3.93276496011559, + "tokens_seen": 479330304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316750250752257, + "loss": 1.3555, + "theoretical_loss": 3.932648304940395, + "tokens_seen": 479461376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316549648946841, + "loss": 1.2894, + "theoretical_loss": 3.9325316905778047, + "tokens_seen": 479592448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043163490471414244, + "loss": 1.3105, + "theoretical_loss": 3.932415117002391, + "tokens_seen": 479723520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043161484453360085, + "loss": 1.4595, + "theoretical_loss": 3.932298584188751, + "tokens_seen": 479854592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315947843530592, + "loss": 1.4289, + "theoretical_loss": 3.9321820921115034, + "tokens_seen": 479985664 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.31689882278442383, + "objective/train/avg_token_score": 0.03435559570789337, + "objective/train/docs_used": 283309, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2388397455215454, + "objective/train/original_loss": 4.061135768890381, + "objective/train/theoretical_loss": 3.932123861341101, + "objective/train/tokens_used": 480051200, + "objective/train/unlikelihood_loss": 0.1754794418811798, + "theoretical_loss": 3.932123861341101, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315747241725176, + "loss": 1.4026, + "theoretical_loss": 3.932065640745289, + "tokens_seen": 480116736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043155466399197594, + "loss": 1.4035, + "theoretical_loss": 3.9319492300647725, + "tokens_seen": 480247808 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043153460381143436, + "loss": 1.4297, + "theoretical_loss": 3.9318328600446413, + "tokens_seen": 480378880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315145436308927, + "loss": 1.2719, + "theoretical_loss": 3.931716530659604, + "tokens_seen": 480509952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314944834503511, + "loss": 1.4338, + "theoretical_loss": 3.9316002418843934, + "tokens_seen": 480641024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043147442326980944, + "loss": 1.4461, + "theoretical_loss": 3.931483993693764, + "tokens_seen": 480772096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314543630892678, + "loss": 1.3509, + "theoretical_loss": 3.9313677860624936, + "tokens_seen": 480903168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314343029087262, + "loss": 1.3157, + "theoretical_loss": 3.9312516189653817, + "tokens_seen": 481034240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314142427281846, + "loss": 1.5189, + "theoretical_loss": 3.9311354923772504, + "tokens_seen": 481165312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043139418254764295, + "loss": 1.4483, + "theoretical_loss": 3.931019406272945, + "tokens_seen": 481296384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313741223671013, + "loss": 1.3292, + "theoretical_loss": 3.9309033606273327, + "tokens_seen": 481427456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313540621865597, + "loss": 1.3071, + "theoretical_loss": 3.930787355415304, + "tokens_seen": 481558528 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4172165095806122, + "objective/train/avg_token_score": 0.026619194075465202, + "objective/train/docs_used": 283520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5104230642318726, + "objective/train/original_loss": 3.943290948867798, + "objective/train/theoretical_loss": 3.93067139061177, + "objective/train/tokens_used": 481689600, + "objective/train/unlikelihood_loss": 0.14345265924930573, + "theoretical_loss": 3.93067139061177, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313340020060181, + "loss": 1.6205, + "theoretical_loss": 3.93067139061177, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043131394182547645, + "loss": 1.3794, + "theoretical_loss": 3.9305554661916657, + "tokens_seen": 481820672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312938816449348, + "loss": 1.563, + "theoretical_loss": 3.9304395821299485, + "tokens_seen": 481951744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127382146439323, + "loss": 1.543, + "theoretical_loss": 3.930323738401597, + "tokens_seen": 482082816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312537612838516, + "loss": 1.3568, + "theoretical_loss": 3.930207934981614, + "tokens_seen": 482213888 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043123370110330995, + "loss": 1.5519, + "theoretical_loss": 3.930092171845022, + "tokens_seen": 482344960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312136409227683, + "loss": 1.4007, + "theoretical_loss": 3.929976448966868, + "tokens_seen": 482476032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311935807422267, + "loss": 1.3958, + "theoretical_loss": 3.92986076632222, + "tokens_seen": 482607104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311735205616851, + "loss": 1.3824, + "theoretical_loss": 3.9297451238861685, + "tokens_seen": 482738176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043115346038114346, + "loss": 1.4257, + "theoretical_loss": 3.929629521633827, + "tokens_seen": 482869248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311334002006018, + "loss": 1.5184, + "theoretical_loss": 3.92951395954033, + "tokens_seen": 483000320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311133400200602, + "loss": 1.402, + "theoretical_loss": 3.9293984375808337, + "tokens_seen": 483131392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310932798395186, + "loss": 1.5764, + "theoretical_loss": 3.929282955730519, + "tokens_seen": 483262464 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4565615952014923, + "objective/train/avg_token_score": 0.03555590286850929, + "objective/train/docs_used": 284734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5730347633361816, + "objective/train/original_loss": 4.092532157897949, + "objective/train/theoretical_loss": 3.9292252298385533, + "objective/train/tokens_used": 483328000, + "objective/train/unlikelihood_loss": 0.2168586552143097, + "theoretical_loss": 3.9292252298385533, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043107321965897696, + "loss": 1.3438, + "theoretical_loss": 3.929167513964585, + "tokens_seen": 483393536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310531594784353, + "loss": 1.3901, + "theoretical_loss": 3.9290521122582573, + "tokens_seen": 483524608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310330992978937, + "loss": 1.3786, + "theoretical_loss": 3.928936750586779, + "tokens_seen": 483655680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043101303911735205, + "loss": 1.4429, + "theoretical_loss": 3.9288214289254193, + "tokens_seen": 483786752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043099297893681046, + "loss": 1.2574, + "theoretical_loss": 3.928706147249466, + "tokens_seen": 483917824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309729187562688, + "loss": 1.2796, + "theoretical_loss": 3.9285909055342314, + "tokens_seen": 484048896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309528585757272, + "loss": 1.4361, + "theoretical_loss": 3.928475703755048, + "tokens_seen": 484179968 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043093279839518555, + "loss": 1.5195, + "theoretical_loss": 3.9283605418872716, + "tokens_seen": 484311040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043091273821464397, + "loss": 1.5687, + "theoretical_loss": 3.9282454199062777, + "tokens_seen": 484442112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043089267803410233, + "loss": 1.5128, + "theoretical_loss": 3.928130337787467, + "tokens_seen": 484573184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308726178535607, + "loss": 1.4754, + "theoretical_loss": 3.9280152955062584, + "tokens_seen": 484704256 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043085255767301905, + "loss": 1.3141, + "theoretical_loss": 3.9279002930380953, + "tokens_seen": 484835328 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4302908182144165, + "objective/train/avg_token_score": 0.014568953774869442, + "objective/train/docs_used": 285372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4252182245254517, + "objective/train/original_loss": 3.9690096378326416, + "objective/train/theoretical_loss": 3.927785330358441, + "objective/train/tokens_used": 484966400, + "objective/train/unlikelihood_loss": 0.10961988568305969, + "theoretical_loss": 3.927785330358441, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308324974924774, + "loss": 1.2884, + "theoretical_loss": 3.927785330358441, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043081243731193583, + "loss": 1.5775, + "theoretical_loss": 3.9276704074427826, + "tokens_seen": 485097472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307923771313942, + "loss": 1.4638, + "theoretical_loss": 3.9275555242666265, + "tokens_seen": 485228544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043077231695085256, + "loss": 1.4386, + "theoretical_loss": 3.927440680805503, + "tokens_seen": 485359616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307522567703109, + "loss": 1.4193, + "theoretical_loss": 3.9273258770349626, + "tokens_seen": 485490688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043073219658976933, + "loss": 1.6056, + "theoretical_loss": 3.9272111129305785, + "tokens_seen": 485621760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307121364092277, + "loss": 1.4051, + "theoretical_loss": 3.9270963884679437, + "tokens_seen": 485752832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043069207622868606, + "loss": 1.2638, + "theoretical_loss": 3.9269817036226757, + "tokens_seen": 485883904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306720160481444, + "loss": 1.3818, + "theoretical_loss": 3.9268670583704113, + "tokens_seen": 486014976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306519558676028, + "loss": 1.2313, + "theoretical_loss": 3.9267524526868085, + "tokens_seen": 486146048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306318956870612, + "loss": 1.4043, + "theoretical_loss": 3.9266378865475495, + "tokens_seen": 486277120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043061183550651956, + "loss": 1.3174, + "theoretical_loss": 3.926523359928335, + "tokens_seen": 486408192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305917753259779, + "loss": 1.4299, + "theoretical_loss": 3.926408872804889, + "tokens_seen": 486539264 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4375610947608948, + "objective/train/avg_token_score": 0.00509691471233964, + "objective/train/docs_used": 285999, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.344993233680725, + "objective/train/original_loss": 3.588568925857544, + "objective/train/theoretical_loss": 3.9263516440464987, + "objective/train/tokens_used": 486604800, + "objective/train/unlikelihood_loss": 0.1715584695339203, + "theoretical_loss": 3.9263516440464987, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305717151454363, + "loss": 1.468, + "theoretical_loss": 3.926294425152957, + "tokens_seen": 486670336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305516549648947, + "loss": 1.5126, + "theoretical_loss": 3.9261800169483037, + "tokens_seen": 486801408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043053159478435306, + "loss": 1.3765, + "theoretical_loss": 3.926065648166719, + "tokens_seen": 486932480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305115346038114, + "loss": 1.6243, + "theoretical_loss": 3.9259513187840103, + "tokens_seen": 487063552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304914744232698, + "loss": 1.4171, + "theoretical_loss": 3.9258370287760087, + "tokens_seen": 487194624 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043047141424272815, + "loss": 1.348, + "theoretical_loss": 3.9257227781185655, + "tokens_seen": 487325696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043045135406218657, + "loss": 1.4053, + "theoretical_loss": 3.925608566787555, + "tokens_seen": 487456768 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043043129388164493, + "loss": 1.3852, + "theoretical_loss": 3.9254943947588705, + "tokens_seen": 487587840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304112337011033, + "loss": 1.3728, + "theoretical_loss": 3.925380262008428, + "tokens_seen": 487718912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043039117352056165, + "loss": 1.1108, + "theoretical_loss": 3.925266168512165, + "tokens_seen": 487849984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043037111334002007, + "loss": 1.584, + "theoretical_loss": 3.9251521142460377, + "tokens_seen": 487981056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043035105315947843, + "loss": 1.4063, + "theoretical_loss": 3.925038099186027, + "tokens_seen": 488112128 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.26423507928848267, + "objective/train/avg_token_score": 0.009158661589026451, + "objective/train/docs_used": 287428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9026793837547302, + "objective/train/original_loss": 3.2273669242858887, + "objective/train/theoretical_loss": 3.9249241233081333, + "objective/train/tokens_used": 488243200, + "objective/train/unlikelihood_loss": 0.31226837635040283, + "theoretical_loss": 3.9249241233081333, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303309929789368, + "loss": 1.4517, + "theoretical_loss": 3.9249241233081333, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043031093279839516, + "loss": 1.5534, + "theoretical_loss": 3.9248101865883767, + "tokens_seen": 488374272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302908726178535, + "loss": 1.4797, + "theoretical_loss": 3.924696289002801, + "tokens_seen": 488505344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043027081243731194, + "loss": 1.4941, + "theoretical_loss": 3.9245824305274697, + "tokens_seen": 488636416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302507522567703, + "loss": 1.4144, + "theoretical_loss": 3.924468611138467, + "tokens_seen": 488767488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043023069207622866, + "loss": 1.4611, + "theoretical_loss": 3.9243548308118994, + "tokens_seen": 488898560 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430210631895687, + "loss": 1.4122, + "theoretical_loss": 3.9242410895238935, + "tokens_seen": 489029632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043019057171514544, + "loss": 1.3031, + "theoretical_loss": 3.9241273872505973, + "tokens_seen": 489160704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043017051153460386, + "loss": 1.4607, + "theoretical_loss": 3.9240137239681787, + "tokens_seen": 489291776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301504513540622, + "loss": 1.5392, + "theoretical_loss": 3.923900099652828, + "tokens_seen": 489422848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301303911735206, + "loss": 1.5051, + "theoretical_loss": 3.9237865142807555, + "tokens_seen": 489553920 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430110330992979, + "loss": 1.6593, + "theoretical_loss": 3.9236729678281925, + "tokens_seen": 489684992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043009027081243736, + "loss": 1.5331, + "theoretical_loss": 3.9235594602713926, + "tokens_seen": 489816064 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.5066593289375305, + "objective/train/avg_token_score": 0.013708590529859066, + "objective/train/docs_used": 288116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7811899185180664, + "objective/train/original_loss": 3.8481087684631348, + "objective/train/theoretical_loss": 3.923502721071488, + "objective/train/tokens_used": 489881600, + "objective/train/unlikelihood_loss": 0.17132464051246643, + "theoretical_loss": 3.923502721071488, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300702106318957, + "loss": 1.5335, + "theoretical_loss": 3.9234459915866275, + "tokens_seen": 489947136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300501504513541, + "loss": 1.4411, + "theoretical_loss": 3.923332561750192, + "tokens_seen": 490078208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043003009027081245, + "loss": 1.466, + "theoretical_loss": 3.9232191707384, + "tokens_seen": 490209280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043001003009027086, + "loss": 1.3247, + "theoretical_loss": 3.9231058185275884, + "tokens_seen": 490340352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299899699097292, + "loss": 1.5211, + "theoretical_loss": 3.922992505094112, + "tokens_seen": 490471424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299699097291876, + "loss": 1.2627, + "theoretical_loss": 3.922879230414349, + "tokens_seen": 490602496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042994984954864595, + "loss": 1.5509, + "theoretical_loss": 3.9227659944646964, + "tokens_seen": 490733568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042992978936810437, + "loss": 1.504, + "theoretical_loss": 3.922652797221573, + "tokens_seen": 490864640 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042990972918756273, + "loss": 1.395, + "theoretical_loss": 3.922539638661418, + "tokens_seen": 490995712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298896690070211, + "loss": 1.5169, + "theoretical_loss": 3.92242651876069, + "tokens_seen": 491126784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042986960882647945, + "loss": 1.8001, + "theoretical_loss": 3.922313437495871, + "tokens_seen": 491257856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298495486459378, + "loss": 1.3611, + "theoretical_loss": 3.9222003948434603, + "tokens_seen": 491388928 + }, + { + "debugging/length": 516.8571428571429, + "debugging/num_segments": 7, + "epoch": 0.15, + "objective/train/aligned_token_count": 0.45747801661491394, + "objective/train/avg_token_score": 0.004869875498116016, + "objective/train/docs_used": 289197, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5986454486846924, + "objective/train/original_loss": 4.020472526550293, + "objective/train/theoretical_loss": 3.92208739077998, + "objective/train/tokens_used": 491520000, + "objective/train/unlikelihood_loss": 0.21023108065128326, + "theoretical_loss": 3.92208739077998, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042982948846539623, + "loss": 1.4155, + "theoretical_loss": 3.92208739077998, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298094282848546, + "loss": 1.5174, + "theoretical_loss": 3.921974425281972, + "tokens_seen": 491651072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042978936810431296, + "loss": 1.3887, + "theoretical_loss": 3.921861498325998, + "tokens_seen": 491782144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297693079237713, + "loss": 1.4294, + "theoretical_loss": 3.9217486098886427, + "tokens_seen": 491913216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042974924774322973, + "loss": 1.6334, + "theoretical_loss": 3.9216357599465073, + "tokens_seen": 492044288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297291875626881, + "loss": 1.2926, + "theoretical_loss": 3.921522948476217, + "tokens_seen": 492175360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042970912738214646, + "loss": 1.4125, + "theoretical_loss": 3.921410175454416, + "tokens_seen": 492306432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296890672016048, + "loss": 1.4243, + "theoretical_loss": 3.9212974408577685, + "tokens_seen": 492437504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296690070210632, + "loss": 1.3942, + "theoretical_loss": 3.92118474466296, + "tokens_seen": 492568576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296489468405216, + "loss": 1.4767, + "theoretical_loss": 3.9210720868466953, + "tokens_seen": 492699648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042962888665997996, + "loss": 1.5349, + "theoretical_loss": 3.9209594673857, + "tokens_seen": 492830720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296088264794383, + "loss": 1.4584, + "theoretical_loss": 3.9208468862567205, + "tokens_seen": 492961792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295887662988967, + "loss": 1.2407, + "theoretical_loss": 3.920734343436523, + "tokens_seen": 493092864 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.41605570912361145, + "objective/train/avg_token_score": 0.049754299223423004, + "objective/train/docs_used": 289918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4484484195709229, + "objective/train/original_loss": 4.115923881530762, + "objective/train/theoretical_loss": 3.920678086384963, + "objective/train/tokens_used": 493158400, + "objective/train/unlikelihood_loss": 0.13206537067890167, + "theoretical_loss": 3.920678086384963, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295687061183551, + "loss": 1.3909, + "theoretical_loss": 3.9206218389018943, + "tokens_seen": 493223936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042954864593781347, + "loss": 1.6181, + "theoretical_loss": 3.9205093726296405, + "tokens_seen": 493355008 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042952858575727183, + "loss": 1.4534, + "theoretical_loss": 3.9203969445965887, + "tokens_seen": 493486080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295085255767302, + "loss": 1.4879, + "theoretical_loss": 3.9202845547795864, + "tokens_seen": 493617152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042948846539618855, + "loss": 1.5724, + "theoretical_loss": 3.9201722031555004, + "tokens_seen": 493748224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042946840521564697, + "loss": 1.5877, + "theoretical_loss": 3.9200598897012187, + "tokens_seen": 493879296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042944834503510533, + "loss": 1.6108, + "theoretical_loss": 3.9199476143936485, + "tokens_seen": 494010368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294282848545637, + "loss": 1.4842, + "theoretical_loss": 3.9198353772097168, + "tokens_seen": 494141440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042940822467402206, + "loss": 1.3961, + "theoretical_loss": 3.9197231781263726, + "tokens_seen": 494272512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042938816449348047, + "loss": 1.4718, + "theoretical_loss": 3.919611017120583, + "tokens_seen": 494403584 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042936810431293883, + "loss": 1.172, + "theoretical_loss": 3.9194988941693354, + "tokens_seen": 494534656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293480441323972, + "loss": 1.5461, + "theoretical_loss": 3.9193868092496382, + "tokens_seen": 494665728 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.38336998224258423, + "objective/train/avg_token_score": 0.007834085263311863, + "objective/train/docs_used": 291263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3380773067474365, + "objective/train/original_loss": 3.855975866317749, + "objective/train/theoretical_loss": 3.919274762338519, + "objective/train/tokens_used": 494796800, + "objective/train/unlikelihood_loss": 0.20109692215919495, + "theoretical_loss": 3.919274762338519, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042932798395185556, + "loss": 1.6039, + "theoretical_loss": 3.919274762338519, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293079237713139, + "loss": 1.2938, + "theoretical_loss": 3.9191627534130253, + "tokens_seen": 494927872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042928786359077234, + "loss": 1.5174, + "theoretical_loss": 3.919050782450225, + "tokens_seen": 495058944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292678034102307, + "loss": 1.3587, + "theoretical_loss": 3.9189388494272057, + "tokens_seen": 495190016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042924774322968906, + "loss": 1.4829, + "theoretical_loss": 3.918826954321074, + "tokens_seen": 495321088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292276830491474, + "loss": 1.5397, + "theoretical_loss": 3.9187150971089584, + "tokens_seen": 495452160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042920762286860584, + "loss": 1.445, + "theoretical_loss": 3.9186032777680055, + "tokens_seen": 495583232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291875626880642, + "loss": 1.6188, + "theoretical_loss": 3.918491496275383, + "tokens_seen": 495714304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042916750250752256, + "loss": 1.4414, + "theoretical_loss": 3.918379752608276, + "tokens_seen": 495845376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291474423269809, + "loss": 1.2727, + "theoretical_loss": 3.9182680467438926, + "tokens_seen": 495976448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291273821464393, + "loss": 1.3956, + "theoretical_loss": 3.918156378659459, + "tokens_seen": 496107520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291073219658977, + "loss": 1.4127, + "theoretical_loss": 3.9180447483322203, + "tokens_seen": 496238592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042908726178535607, + "loss": 1.3812, + "theoretical_loss": 3.9179331557394432, + "tokens_seen": 496369664 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.36589688062667847, + "objective/train/avg_token_score": 0.019114283844828606, + "objective/train/docs_used": 291574, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3350896835327148, + "objective/train/original_loss": 4.14976167678833, + "objective/train/theoretical_loss": 3.9178773735863786, + "objective/train/tokens_used": 496435200, + "objective/train/unlikelihood_loss": 0.12984949350357056, + "theoretical_loss": 3.9178773735863786, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042906720160481443, + "loss": 1.3808, + "theoretical_loss": 3.917821600858413, + "tokens_seen": 496500736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290471414242728, + "loss": 1.5891, + "theoretical_loss": 3.917710083666434, + "tokens_seen": 496631808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290270812437312, + "loss": 1.279, + "theoretical_loss": 3.9175986041408315, + "tokens_seen": 496762880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042900702106318957, + "loss": 1.3422, + "theoretical_loss": 3.9174871622589498, + "tokens_seen": 496893952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042898696088264793, + "loss": 1.4238, + "theoretical_loss": 3.9173757579981534, + "tokens_seen": 497025024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289669007021063, + "loss": 1.4951, + "theoretical_loss": 3.9172643913358254, + "tokens_seen": 497156096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289468405215647, + "loss": 1.4327, + "theoretical_loss": 3.917153062249368, + "tokens_seen": 497287168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289267803410231, + "loss": 1.3294, + "theoretical_loss": 3.9170417707162057, + "tokens_seen": 497418240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042890672016048144, + "loss": 1.5096, + "theoretical_loss": 3.9169305167137796, + "tokens_seen": 497549312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288866599799398, + "loss": 1.587, + "theoretical_loss": 3.9168193002195513, + "tokens_seen": 497680384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042886659979939816, + "loss": 1.3454, + "theoretical_loss": 3.916708121211002, + "tokens_seen": 497811456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288465396188566, + "loss": 1.4182, + "theoretical_loss": 3.916596979665632, + "tokens_seen": 497942528 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.35257819294929504, + "objective/train/avg_token_score": 0.013382690958678722, + "objective/train/docs_used": 293024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3515236377716064, + "objective/train/original_loss": 3.946211576461792, + "objective/train/theoretical_loss": 3.9164858755609613, + "objective/train/tokens_used": 498073600, + "objective/train/unlikelihood_loss": 0.19796288013458252, + "theoretical_loss": 3.9164858755609613, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042882647943831494, + "loss": 1.4908, + "theoretical_loss": 3.9164858755609613, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288064192577733, + "loss": 1.438, + "theoretical_loss": 3.9163748088745303, + "tokens_seen": 498204672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878635907723166, + "loss": 1.3417, + "theoretical_loss": 3.916263779583896, + "tokens_seen": 498335744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287662988966901, + "loss": 1.5937, + "theoretical_loss": 3.916152787666638, + "tokens_seen": 498466816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042874623871614844, + "loss": 1.4663, + "theoretical_loss": 3.9160418331003526, + "tokens_seen": 498597888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287261785356068, + "loss": 1.4601, + "theoretical_loss": 3.9159309158626576, + "tokens_seen": 498728960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042870611835506517, + "loss": 1.2813, + "theoretical_loss": 3.915820035931188, + "tokens_seen": 498860032 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042868605817452353, + "loss": 1.2875, + "theoretical_loss": 3.9157091932836, + "tokens_seen": 498991104 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428665997993982, + "loss": 1.5178, + "theoretical_loss": 3.9155983878975675, + "tokens_seen": 499122176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042864593781344036, + "loss": 1.5188, + "theoretical_loss": 3.9154876197507846, + "tokens_seen": 499253248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286258776328987, + "loss": 1.5098, + "theoretical_loss": 3.9153768888209646, + "tokens_seen": 499384320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286058174523571, + "loss": 1.3121, + "theoretical_loss": 3.915266195085839, + "tokens_seen": 499515392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285857572718155, + "loss": 1.5902, + "theoretical_loss": 3.91515553852316, + "tokens_seen": 499646464 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4228983521461487, + "objective/train/avg_token_score": 0.019886652007699013, + "objective/train/docs_used": 293514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.436700701713562, + "objective/train/original_loss": 3.5851337909698486, + "objective/train/theoretical_loss": 3.91510022417454, + "objective/train/tokens_used": 499712000, + "objective/train/unlikelihood_loss": 0.24415582418441772, + "theoretical_loss": 3.91510022417454, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042856569709127387, + "loss": 1.4769, + "theoretical_loss": 3.915044919110697, + "tokens_seen": 499777536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854563691073223, + "loss": 1.4117, + "theoretical_loss": 3.9149343368262404, + "tokens_seen": 499908608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285255767301906, + "loss": 1.4151, + "theoretical_loss": 3.9148237916475983, + "tokens_seen": 500039680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042850551654964895, + "loss": 1.4644, + "theoretical_loss": 3.914713283552599, + "tokens_seen": 500170752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042848545636910737, + "loss": 1.4707, + "theoretical_loss": 3.914602812519089, + "tokens_seen": 500301824 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846539618856573, + "loss": 1.3893, + "theoretical_loss": 3.914492378524934, + "tokens_seen": 500432896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284453360080241, + "loss": 1.3221, + "theoretical_loss": 3.9143819815480194, + "tokens_seen": 500563968 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042842527582748246, + "loss": 1.4009, + "theoretical_loss": 3.9142716215662494, + "tokens_seen": 500695040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042840521564694087, + "loss": 1.4476, + "theoretical_loss": 3.9141612985575454, + "tokens_seen": 500826112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042838515546639923, + "loss": 1.3708, + "theoretical_loss": 3.91405101249985, + "tokens_seen": 500957184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283650952858576, + "loss": 1.5202, + "theoretical_loss": 3.9139407633711243, + "tokens_seen": 501088256 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042834503510531596, + "loss": 1.393, + "theoretical_loss": 3.9138305511493474, + "tokens_seen": 501219328 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.34488025307655334, + "objective/train/avg_token_score": 0.03448681905865669, + "objective/train/docs_used": 294722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1151436567306519, + "objective/train/original_loss": 3.941049337387085, + "objective/train/theoretical_loss": 3.9137203758125176, + "objective/train/tokens_used": 501350400, + "objective/train/unlikelihood_loss": 0.2037958949804306, + "theoretical_loss": 3.9137203758125176, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283249749247743, + "loss": 1.4004, + "theoretical_loss": 3.9137203758125176, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042830491474423274, + "loss": 1.2935, + "theoretical_loss": 3.913610237338653, + "tokens_seen": 501481472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282848545636911, + "loss": 1.2546, + "theoretical_loss": 3.913500135705789, + "tokens_seen": 501612544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042826479438314946, + "loss": 1.5618, + "theoretical_loss": 3.913390070891981, + "tokens_seen": 501743616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282447342026078, + "loss": 1.3867, + "theoretical_loss": 3.9132800428753027, + "tokens_seen": 501874688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042822467402206624, + "loss": 1.2633, + "theoretical_loss": 3.9131700516338475, + "tokens_seen": 502005760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282046138415246, + "loss": 1.4745, + "theoretical_loss": 3.913060097145725, + "tokens_seen": 502136832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042818455366098296, + "loss": 1.46, + "theoretical_loss": 3.9129501793890666, + "tokens_seen": 502267904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281644934804413, + "loss": 1.2897, + "theoretical_loss": 3.9128402983420214, + "tokens_seen": 502398976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281444332998997, + "loss": 1.2421, + "theoretical_loss": 3.912730453982756, + "tokens_seen": 502530048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281243731193581, + "loss": 1.3616, + "theoretical_loss": 3.912620646289457, + "tokens_seen": 502661120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042810431293881647, + "loss": 1.5323, + "theoretical_loss": 3.9125108752403293, + "tokens_seen": 502792192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042808425275827483, + "loss": 1.2956, + "theoretical_loss": 3.9124011408135964, + "tokens_seen": 502923264 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4918743968009949, + "objective/train/avg_token_score": 0.003718223888427019, + "objective/train/docs_used": 295112, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6487371921539307, + "objective/train/original_loss": 4.021548748016357, + "objective/train/theoretical_loss": 3.9123462873268275, + "objective/train/tokens_used": 502988800, + "objective/train/unlikelihood_loss": 0.127858504652977, + "theoretical_loss": 3.9123462873268275, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280641925777332, + "loss": 1.3411, + "theoretical_loss": 3.9122914429874998, + "tokens_seen": 503054336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280441323971916, + "loss": 1.5446, + "theoretical_loss": 3.9121817817403004, + "tokens_seen": 503185408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802407221664997, + "loss": 1.2649, + "theoretical_loss": 3.912072157050278, + "tokens_seen": 503316480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042800401203610833, + "loss": 1.2767, + "theoretical_loss": 3.91196256889573, + "tokens_seen": 503447552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279839518555667, + "loss": 1.2984, + "theoretical_loss": 3.911853017254973, + "tokens_seen": 503578624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279638916750251, + "loss": 1.3402, + "theoretical_loss": 3.911743502106341, + "tokens_seen": 503709696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279438314944835, + "loss": 1.4761, + "theoretical_loss": 3.9116340234281886, + "tokens_seen": 503840768 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042792377131394184, + "loss": 1.4801, + "theoretical_loss": 3.9115245811988864, + "tokens_seen": 503971840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279037111334002, + "loss": 1.4165, + "theoretical_loss": 3.911415175396825, + "tokens_seen": 504102912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042788365095285856, + "loss": 1.4903, + "theoretical_loss": 3.9113058060004136, + "tokens_seen": 504233984 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427863590772317, + "loss": 1.3463, + "theoretical_loss": 3.911196472988079, + "tokens_seen": 504365056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042784353059177534, + "loss": 1.4082, + "theoretical_loss": 3.911087176338266, + "tokens_seen": 504496128 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.2879398763179779, + "objective/train/avg_token_score": 0.030777588486671448, + "objective/train/docs_used": 296352, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.946451723575592, + "objective/train/original_loss": 3.832697629928589, + "objective/train/theoretical_loss": 3.910977916029439, + "objective/train/tokens_used": 504627200, + "objective/train/unlikelihood_loss": 0.18606476485729218, + "theoretical_loss": 3.910977916029439, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278234704112337, + "loss": 1.31, + "theoretical_loss": 3.910977916029439, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042780341023069206, + "loss": 1.3279, + "theoretical_loss": 3.91086869204008, + "tokens_seen": 504758272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277833500501505, + "loss": 1.4205, + "theoretical_loss": 3.9107595043486905, + "tokens_seen": 504889344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042776328986960884, + "loss": 1.5258, + "theoretical_loss": 3.9106503529337875, + "tokens_seen": 505020416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277432296890672, + "loss": 1.5069, + "theoretical_loss": 3.910541237773909, + "tokens_seen": 505151488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042772316950852557, + "loss": 1.5058, + "theoretical_loss": 3.9104321588476108, + "tokens_seen": 505282560 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042770310932798393, + "loss": 1.4835, + "theoretical_loss": 3.9103231161334655, + "tokens_seen": 505413632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042768304914744235, + "loss": 1.3955, + "theoretical_loss": 3.910214109610065, + "tokens_seen": 505544704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276629889669007, + "loss": 1.4016, + "theoretical_loss": 3.9101051392560198, + "tokens_seen": 505675776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042764292878635907, + "loss": 1.4402, + "theoretical_loss": 3.9099962050499584, + "tokens_seen": 505806848 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042762286860581743, + "loss": 1.4021, + "theoretical_loss": 3.9098873069705267, + "tokens_seen": 505937920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042760280842527585, + "loss": 1.4597, + "theoretical_loss": 3.909778444996389, + "tokens_seen": 506068992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275827482447342, + "loss": 1.469, + "theoretical_loss": 3.9096696191062277, + "tokens_seen": 506200064 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.37237292528152466, + "objective/train/avg_token_score": 0.01380022894591093, + "objective/train/docs_used": 297118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3744415044784546, + "objective/train/original_loss": 4.307300090789795, + "objective/train/theoretical_loss": 3.909615219685982, + "objective/train/tokens_used": 506265600, + "objective/train/unlikelihood_loss": 0.17518089711666107, + "theoretical_loss": 3.909615219685982, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275626880641926, + "loss": 1.4586, + "theoretical_loss": 3.9095608292787443, + "tokens_seen": 506331136 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042754262788365094, + "loss": 1.4717, + "theoretical_loss": 3.9094520754926574, + "tokens_seen": 506462208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275225677031093, + "loss": 1.4785, + "theoretical_loss": 3.909343357726703, + "tokens_seen": 506593280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275025075225677, + "loss": 1.4522, + "theoretical_loss": 3.9092346759596373, + "tokens_seen": 506724352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274824473420261, + "loss": 1.4267, + "theoretical_loss": 3.9091260301702326, + "tokens_seen": 506855424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042746238716148444, + "loss": 1.6183, + "theoretical_loss": 3.90901742033728, + "tokens_seen": 506986496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274423269809428, + "loss": 1.4882, + "theoretical_loss": 3.9089088464395876, + "tokens_seen": 507117568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274222668004012, + "loss": 1.3987, + "theoretical_loss": 3.9088003084559833, + "tokens_seen": 507248640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274022066198596, + "loss": 1.6277, + "theoretical_loss": 3.9086918063653115, + "tokens_seen": 507379712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042738214643931794, + "loss": 1.5008, + "theoretical_loss": 3.9085833401464347, + "tokens_seen": 507510784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273620862587763, + "loss": 1.4856, + "theoretical_loss": 3.908474909778234, + "tokens_seen": 507641856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042734202607823467, + "loss": 1.3557, + "theoretical_loss": 3.9083665152396074, + "tokens_seen": 507772928 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.35135629773139954, + "objective/train/avg_token_score": 0.02974272146821022, + "objective/train/docs_used": 298304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.351550579071045, + "objective/train/original_loss": 4.083728790283203, + "objective/train/theoretical_loss": 3.908258156509472, + "objective/train/tokens_used": 507904000, + "objective/train/unlikelihood_loss": 0.15935710072517395, + "theoretical_loss": 3.908258156509472, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273219658976931, + "loss": 1.406, + "theoretical_loss": 3.908258156509472, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042730190571715145, + "loss": 1.5871, + "theoretical_loss": 3.9081498335667613, + "tokens_seen": 508035072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272818455366098, + "loss": 1.2993, + "theoretical_loss": 3.908041546390428, + "tokens_seen": 508166144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042726178535606817, + "loss": 1.4989, + "theoretical_loss": 3.9079332949594408, + "tokens_seen": 508297216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272417251755266, + "loss": 1.417, + "theoretical_loss": 3.907825079252789, + "tokens_seen": 508428288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042722166499498495, + "loss": 1.4882, + "theoretical_loss": 3.907716899249477, + "tokens_seen": 508559360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272016048144433, + "loss": 1.4883, + "theoretical_loss": 3.9076087549285274, + "tokens_seen": 508690432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271815446339017, + "loss": 1.5095, + "theoretical_loss": 3.9075006462689825, + "tokens_seen": 508821504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271614844533601, + "loss": 1.4736, + "theoretical_loss": 3.9073925732498997, + "tokens_seen": 508952576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271414242728185, + "loss": 1.4869, + "theoretical_loss": 3.907284535850356, + "tokens_seen": 509083648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042712136409227687, + "loss": 1.401, + "theoretical_loss": 3.9071765340494453, + "tokens_seen": 509214720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042710130391173523, + "loss": 1.3473, + "theoretical_loss": 3.907068567826278, + "tokens_seen": 509345792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270812437311936, + "loss": 1.3851, + "theoretical_loss": 3.906960637159985, + "tokens_seen": 509476864 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.29997557401657104, + "objective/train/avg_token_score": 0.05142614245414734, + "objective/train/docs_used": 298986, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.076449990272522, + "objective/train/original_loss": 4.7175726890563965, + "objective/train/theoretical_loss": 3.9069066851541487, + "objective/train/tokens_used": 509542400, + "objective/train/unlikelihood_loss": 0.11390677094459534, + "theoretical_loss": 3.9069066851541487, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427061183550652, + "loss": 1.4249, + "theoretical_loss": 3.906852742029712, + "tokens_seen": 509607936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042704112337011037, + "loss": 1.4852, + "theoretical_loss": 3.9067448824146243, + "tokens_seen": 509739008 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042702106318956873, + "loss": 1.411, + "theoretical_loss": 3.906637058293903, + "tokens_seen": 509870080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270010030090271, + "loss": 1.31, + "theoretical_loss": 3.9065292696467475, + "tokens_seen": 510001152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269809428284855, + "loss": 1.5244, + "theoretical_loss": 3.906421516452376, + "tokens_seen": 510132224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269608826479439, + "loss": 1.5396, + "theoretical_loss": 3.9063137986900216, + "tokens_seen": 510263296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042694082246740224, + "loss": 1.4904, + "theoretical_loss": 3.9062061163389377, + "tokens_seen": 510394368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269207622868606, + "loss": 1.4771, + "theoretical_loss": 3.906098469378393, + "tokens_seen": 510525440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042690070210631896, + "loss": 1.4655, + "theoretical_loss": 3.905990857787675, + "tokens_seen": 510656512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268806419257774, + "loss": 1.5493, + "theoretical_loss": 3.905883281546088, + "tokens_seen": 510787584 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042686058174523574, + "loss": 1.604, + "theoretical_loss": 3.9057757406329534, + "tokens_seen": 510918656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268405215646941, + "loss": 1.4769, + "theoretical_loss": 3.9056682350276106, + "tokens_seen": 511049728 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.5116080045700073, + "objective/train/avg_token_score": 0.008771687746047974, + "objective/train/docs_used": 299996, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8161802291870117, + "objective/train/original_loss": 3.781794548034668, + "objective/train/theoretical_loss": 3.905560764709417, + "objective/train/tokens_used": 511180800, + "objective/train/unlikelihood_loss": 0.15718472003936768, + "theoretical_loss": 3.905560764709417, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042682046138415246, + "loss": 1.3467, + "theoretical_loss": 3.905560764709417, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268004012036109, + "loss": 1.2939, + "theoretical_loss": 3.905453329657745, + "tokens_seen": 511311872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042678034102306924, + "loss": 1.4485, + "theoretical_loss": 3.9053459298519875, + "tokens_seen": 511442944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267602808425276, + "loss": 1.3541, + "theoretical_loss": 3.905238565271553, + "tokens_seen": 511574016 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042674022066198597, + "loss": 1.4509, + "theoretical_loss": 3.9051312358958663, + "tokens_seen": 511705088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042672016048144433, + "loss": 1.3144, + "theoretical_loss": 3.905023941704371, + "tokens_seen": 511836160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042670010030090275, + "loss": 1.5859, + "theoretical_loss": 3.9049166826765283, + "tokens_seen": 511967232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266800401203611, + "loss": 1.4643, + "theoretical_loss": 3.9048094587918154, + "tokens_seen": 512098304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042665997993981947, + "loss": 1.62, + "theoretical_loss": 3.904702270029727, + "tokens_seen": 512229376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042663991975927783, + "loss": 1.474, + "theoretical_loss": 3.9045951163697756, + "tokens_seen": 512360448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042661985957873625, + "loss": 1.4682, + "theoretical_loss": 3.904487997791491, + "tokens_seen": 512491520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265997993981946, + "loss": 1.313, + "theoretical_loss": 3.904380914274419, + "tokens_seen": 512622592 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426579739217653, + "loss": 1.5021, + "theoretical_loss": 3.9042738657981237, + "tokens_seen": 512753664 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.28787878155708313, + "objective/train/avg_token_score": 0.02153957076370716, + "objective/train/docs_used": 300643, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0317378044128418, + "objective/train/original_loss": 4.142201900482178, + "objective/train/theoretical_loss": 3.9042203546938854, + "objective/train/tokens_used": 512819200, + "objective/train/unlikelihood_loss": 0.20485782623291016, + "theoretical_loss": 3.9042203546938854, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042655967903711134, + "loss": 1.438, + "theoretical_loss": 3.9041668523421853, + "tokens_seen": 512884736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265396188565697, + "loss": 1.5775, + "theoretical_loss": 3.9040598738862027, + "tokens_seen": 513015808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265195586760281, + "loss": 1.4891, + "theoretical_loss": 3.90395293040979, + "tokens_seen": 513146880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264994984954865, + "loss": 1.6842, + "theoretical_loss": 3.90384602189258, + "tokens_seen": 513277952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042647943831494484, + "loss": 1.5024, + "theoretical_loss": 3.9037391483142208, + "tokens_seen": 513409024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264593781344032, + "loss": 1.6385, + "theoretical_loss": 3.9036323096543795, + "tokens_seen": 513540096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264393179538616, + "loss": 1.5277, + "theoretical_loss": 3.9035255058927394, + "tokens_seen": 513671168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042641925777332, + "loss": 1.5275, + "theoretical_loss": 3.903418737009, + "tokens_seen": 513802240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042639919759277834, + "loss": 1.3673, + "theoretical_loss": 3.903312002982879, + "tokens_seen": 513933312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263791374122367, + "loss": 1.2568, + "theoretical_loss": 3.9032053037941106, + "tokens_seen": 514064384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042635907723169507, + "loss": 1.3819, + "theoretical_loss": 3.903098639422445, + "tokens_seen": 514195456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263390170511535, + "loss": 1.5151, + "theoretical_loss": 3.902992009847652, + "tokens_seen": 514326528 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.45747801661491394, + "objective/train/avg_token_score": 0.014842421747744083, + "objective/train/docs_used": 301818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6070836782455444, + "objective/train/original_loss": 3.8919403553009033, + "objective/train/theoretical_loss": 3.9028854150495143, + "objective/train/tokens_used": 514457600, + "objective/train/unlikelihood_loss": 0.16238826513290405, + "theoretical_loss": 3.9028854150495143, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042631895687061185, + "loss": 1.4055, + "theoretical_loss": 3.9028854150495143, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262988966900702, + "loss": 1.4375, + "theoretical_loss": 3.9027788550078357, + "tokens_seen": 514588672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042627883650952857, + "loss": 1.461, + "theoretical_loss": 3.9026723297024337, + "tokens_seen": 514719744 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426258776328987, + "loss": 1.4347, + "theoretical_loss": 3.9025658391131444, + "tokens_seen": 514850816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042623871614844535, + "loss": 1.5544, + "theoretical_loss": 3.9024593832198198, + "tokens_seen": 514981888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262186559679037, + "loss": 1.4037, + "theoretical_loss": 3.9023529620023294, + "tokens_seen": 515112960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261985957873621, + "loss": 1.4876, + "theoretical_loss": 3.902246575440559, + "tokens_seen": 515244032 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042617853560682044, + "loss": 1.3702, + "theoretical_loss": 3.9021402235144116, + "tokens_seen": 515375104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042615847542627885, + "loss": 1.3778, + "theoretical_loss": 3.9020339062038065, + "tokens_seen": 515506176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261384152457372, + "loss": 1.4036, + "theoretical_loss": 3.90192762348868, + "tokens_seen": 515637248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261183550651956, + "loss": 1.4649, + "theoretical_loss": 3.901821375348985, + "tokens_seen": 515768320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042609829488465394, + "loss": 1.5229, + "theoretical_loss": 3.9017151617646917, + "tokens_seen": 515899392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042607823470411236, + "loss": 1.5503, + "theoretical_loss": 3.9016089827157865, + "tokens_seen": 516030464 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.44012707471847534, + "objective/train/avg_token_score": 0.02129729837179184, + "objective/train/docs_used": 302480, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6454428434371948, + "objective/train/original_loss": 4.097105026245117, + "objective/train/theoretical_loss": 3.9015559061358545, + "objective/train/tokens_used": 516096000, + "objective/train/unlikelihood_loss": 0.17164163291454315, + "theoretical_loss": 3.9015559061358545, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260581745235707, + "loss": 1.5246, + "theoretical_loss": 3.901502838182272, + "tokens_seen": 516161536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260381143430291, + "loss": 1.397, + "theoretical_loss": 3.9013967281441677, + "tokens_seen": 516292608 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042601805416248744, + "loss": 1.4353, + "theoretical_loss": 3.901290652581511, + "tokens_seen": 516423680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259979939819458, + "loss": 1.5998, + "theoretical_loss": 3.9011846114743536, + "tokens_seen": 516554752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259779338014042, + "loss": 1.5021, + "theoretical_loss": 3.9010786048027666, + "tokens_seen": 516685824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259578736208626, + "loss": 1.5956, + "theoretical_loss": 3.900972632546835, + "tokens_seen": 516816896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042593781344032095, + "loss": 1.5683, + "theoretical_loss": 3.9008666946866613, + "tokens_seen": 516947968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259177532597793, + "loss": 1.486, + "theoretical_loss": 3.900760791202366, + "tokens_seen": 517079040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258976930792377, + "loss": 1.378, + "theoretical_loss": 3.9006549220740836, + "tokens_seen": 517210112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258776328986961, + "loss": 1.4012, + "theoretical_loss": 3.9005490872819673, + "tokens_seen": 517341184 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042585757271815445, + "loss": 1.3931, + "theoretical_loss": 3.9004432868061856, + "tokens_seen": 517472256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258375125376128, + "loss": 1.5117, + "theoretical_loss": 3.9003375206269233, + "tokens_seen": 517603328 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3442082107067108, + "objective/train/avg_token_score": 0.01602189801633358, + "objective/train/docs_used": 303675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2318471670150757, + "objective/train/original_loss": 4.042038440704346, + "objective/train/theoretical_loss": 3.9002317887243834, + "objective/train/tokens_used": 517734400, + "objective/train/unlikelihood_loss": 0.17470526695251465, + "theoretical_loss": 3.9002317887243834, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258174523570712, + "loss": 1.4767, + "theoretical_loss": 3.9002317887243834, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257973921765296, + "loss": 1.3376, + "theoretical_loss": 3.900126091078783, + "tokens_seen": 517865472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042577733199598795, + "loss": 1.3587, + "theoretical_loss": 3.9000204276703565, + "tokens_seen": 517996544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257572718154463, + "loss": 1.3704, + "theoretical_loss": 3.8999147984793554, + "tokens_seen": 518127616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257372116349047, + "loss": 1.47, + "theoretical_loss": 3.8998092034860474, + "tokens_seen": 518258688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257171514543631, + "loss": 1.3936, + "theoretical_loss": 3.8997036426707155, + "tokens_seen": 518389760 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042569709127382145, + "loss": 1.4903, + "theoretical_loss": 3.89959811601366, + "tokens_seen": 518520832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256770310932798, + "loss": 1.4924, + "theoretical_loss": 3.899492623495198, + "tokens_seen": 518651904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042565697091273823, + "loss": 1.5132, + "theoretical_loss": 3.8993871650956606, + "tokens_seen": 518782976 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042563691073219665, + "loss": 1.4211, + "theoretical_loss": 3.8992817407953986, + "tokens_seen": 518914048 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425616850551655, + "loss": 1.6041, + "theoretical_loss": 3.8991763505747765, + "tokens_seen": 519045120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255967903711134, + "loss": 1.3711, + "theoretical_loss": 3.899070994414176, + "tokens_seen": 519176192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042557673019057174, + "loss": 1.5216, + "theoretical_loss": 3.8989656722939943, + "tokens_seen": 519307264 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.5273093581199646, + "objective/train/avg_token_score": 0.004553263075649738, + "objective/train/docs_used": 304298, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.785210132598877, + "objective/train/original_loss": 3.7178406715393066, + "objective/train/theoretical_loss": 3.8989130239929395, + "objective/train/tokens_used": 519372800, + "objective/train/unlikelihood_loss": 0.14067475497722626, + "theoretical_loss": 3.8989130239929395, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255566700100301, + "loss": 1.581, + "theoretical_loss": 3.8988603841946463, + "tokens_seen": 519438336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255366098294885, + "loss": 1.3521, + "theoretical_loss": 3.898755130096562, + "tokens_seen": 519569408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255165496489469, + "loss": 1.5186, + "theoretical_loss": 3.8986499099801875, + "tokens_seen": 519700480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549648946840524, + "loss": 1.5265, + "theoretical_loss": 3.8985447238259856, + "tokens_seen": 519831552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254764292878636, + "loss": 1.5886, + "theoretical_loss": 3.8984395716144347, + "tokens_seen": 519962624 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425456369107322, + "loss": 1.2698, + "theoretical_loss": 3.8983344533260307, + "tokens_seen": 520093696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254363089267804, + "loss": 1.4448, + "theoretical_loss": 3.8982293689412835, + "tokens_seen": 520224768 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042541624874623874, + "loss": 1.5163, + "theoretical_loss": 3.898124318440721, + "tokens_seen": 520355840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253961885656971, + "loss": 1.527, + "theoretical_loss": 3.8980193018048857, + "tokens_seen": 520486912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042537612838515547, + "loss": 1.6545, + "theoretical_loss": 3.8979143190143377, + "tokens_seen": 520617984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253560682046139, + "loss": 1.4184, + "theoretical_loss": 3.8978093700496514, + "tokens_seen": 520749056 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533600802407225, + "loss": 1.4471, + "theoretical_loss": 3.897704454891419, + "tokens_seen": 520880128 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.42644184827804565, + "objective/train/avg_token_score": 0.012661627493798733, + "objective/train/docs_used": 305033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3695693016052246, + "objective/train/original_loss": 3.7167294025421143, + "objective/train/theoretical_loss": 3.897599573520247, + "objective/train/tokens_used": 521011200, + "objective/train/unlikelihood_loss": 0.21720042824745178, + "theoretical_loss": 3.897599573520247, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253159478435306, + "loss": 1.4253, + "theoretical_loss": 3.897599573520247, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042529588766298897, + "loss": 1.4934, + "theoretical_loss": 3.8974947259167596, + "tokens_seen": 521142272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252758274824474, + "loss": 1.4735, + "theoretical_loss": 3.8973899120615956, + "tokens_seen": 521273344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042525576730190575, + "loss": 1.4753, + "theoretical_loss": 3.897285131935411, + "tokens_seen": 521404416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252357071213641, + "loss": 1.6052, + "theoretical_loss": 3.897180385518876, + "tokens_seen": 521535488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252156469408225, + "loss": 1.4427, + "theoretical_loss": 3.8970756727926785, + "tokens_seen": 521666560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042519558676028084, + "loss": 1.5753, + "theoretical_loss": 3.896970993737522, + "tokens_seen": 521797632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042517552657973925, + "loss": 1.5369, + "theoretical_loss": 3.8968663483341253, + "tokens_seen": 521928704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251554663991976, + "loss": 1.3264, + "theoretical_loss": 3.8967617365632217, + "tokens_seen": 522059776 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425135406218656, + "loss": 1.3828, + "theoretical_loss": 3.8966571584055645, + "tokens_seen": 522190848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042511534603811434, + "loss": 1.5941, + "theoretical_loss": 3.8965526138419184, + "tokens_seen": 522321920 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042509528585757276, + "loss": 1.284, + "theoretical_loss": 3.896448102853067, + "tokens_seen": 522452992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250752256770311, + "loss": 1.4964, + "theoretical_loss": 3.8963436254198074, + "tokens_seen": 522584064 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4902859330177307, + "objective/train/avg_token_score": 0.007528013549745083, + "objective/train/docs_used": 306243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7427791357040405, + "objective/train/original_loss": 3.7014918327331543, + "objective/train/theoretical_loss": 3.8962913992805284, + "objective/train/tokens_used": 522649600, + "objective/train/unlikelihood_loss": 0.16139429807662964, + "theoretical_loss": 3.8962913992805284, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250551654964895, + "loss": 1.5339, + "theoretical_loss": 3.896239181522954, + "tokens_seen": 522715136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042503510531594784, + "loss": 1.5216, + "theoretical_loss": 3.8961347711433376, + "tokens_seen": 522846208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250150451354062, + "loss": 1.4623, + "theoretical_loss": 3.8960303942618024, + "tokens_seen": 522977280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249949849548646, + "loss": 1.346, + "theoretical_loss": 3.8959260508592104, + "tokens_seen": 523108352 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424974924774323, + "loss": 1.6433, + "theoretical_loss": 3.895821740916439, + "tokens_seen": 523239424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042495486459378135, + "loss": 1.3234, + "theoretical_loss": 3.89571746441438, + "tokens_seen": 523370496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249348044132397, + "loss": 1.546, + "theoretical_loss": 3.8956132213339423, + "tokens_seen": 523501568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249147442326981, + "loss": 1.4065, + "theoretical_loss": 3.8955090116560505, + "tokens_seen": 523632640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248946840521565, + "loss": 1.494, + "theoretical_loss": 3.8954048353616435, + "tokens_seen": 523763712 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042487462387161485, + "loss": 1.6275, + "theoretical_loss": 3.895300692431677, + "tokens_seen": 523894784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248545636910732, + "loss": 1.3638, + "theoretical_loss": 3.895196582847123, + "tokens_seen": 524025856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248345035105316, + "loss": 1.4729, + "theoretical_loss": 3.8950925065889663, + "tokens_seen": 524156928 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4181329309940338, + "objective/train/avg_token_score": 0.01068844273686409, + "objective/train/docs_used": 306972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3113154172897339, + "objective/train/original_loss": 3.7316195964813232, + "objective/train/theoretical_loss": 3.8949884636382106, + "objective/train/tokens_used": 524288000, + "objective/train/unlikelihood_loss": 0.1870473474264145, + "theoretical_loss": 3.8949884636382106, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042481444332999, + "loss": 1.5167, + "theoretical_loss": 3.8949884636382106, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042479438314944835, + "loss": 1.4619, + "theoretical_loss": 3.894884453975873, + "tokens_seen": 524419072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247743229689067, + "loss": 1.4227, + "theoretical_loss": 3.894780477582988, + "tokens_seen": 524550144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247542627883651, + "loss": 1.5495, + "theoretical_loss": 3.8946765344406034, + "tokens_seen": 524681216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247342026078235, + "loss": 1.362, + "theoretical_loss": 3.8945726245297836, + "tokens_seen": 524812288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042471414242728186, + "loss": 1.4983, + "theoretical_loss": 3.894468747831609, + "tokens_seen": 524943360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246940822467402, + "loss": 1.7403, + "theoretical_loss": 3.8943649043271744, + "tokens_seen": 525074432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246740220661986, + "loss": 1.5129, + "theoretical_loss": 3.894261093997592, + "tokens_seen": 525205504 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424653961885657, + "loss": 1.535, + "theoretical_loss": 3.8941573168239874, + "tokens_seen": 525336576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042463390170511536, + "loss": 1.4884, + "theoretical_loss": 3.894053572787502, + "tokens_seen": 525467648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246138415245737, + "loss": 1.6132, + "theoretical_loss": 3.8939498618692934, + "tokens_seen": 525598720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245937813440321, + "loss": 1.5335, + "theoretical_loss": 3.8938461840505347, + "tokens_seen": 525729792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042457372116349044, + "loss": 1.4542, + "theoretical_loss": 3.893742539312414, + "tokens_seen": 525860864 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.43389540910720825, + "objective/train/avg_token_score": 0.013585958629846573, + "objective/train/docs_used": 308263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.377792477607727, + "objective/train/original_loss": 3.8568532466888428, + "objective/train/theoretical_loss": 3.8936907293427176, + "objective/train/tokens_used": 525926400, + "objective/train/unlikelihood_loss": 0.1510537713766098, + "theoretical_loss": 3.8936907293427176, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042455366098294886, + "loss": 1.3444, + "theoretical_loss": 3.893638927636134, + "tokens_seen": 525991936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245336008024072, + "loss": 1.5525, + "theoretical_loss": 3.8935353490029136, + "tokens_seen": 526123008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245135406218656, + "loss": 1.395, + "theoretical_loss": 3.8934318033939874, + "tokens_seen": 526254080 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042449348044132395, + "loss": 1.4722, + "theoretical_loss": 3.8933282907906044, + "tokens_seen": 526385152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042447342026078236, + "loss": 1.4891, + "theoretical_loss": 3.89322481117403, + "tokens_seen": 526516224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244533600802407, + "loss": 1.3796, + "theoretical_loss": 3.893121364525544, + "tokens_seen": 526647296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244332998996991, + "loss": 1.4325, + "theoretical_loss": 3.893017950826441, + "tokens_seen": 526778368 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042441323971915745, + "loss": 1.519, + "theoretical_loss": 3.892914570058033, + "tokens_seen": 526909440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243931795386158, + "loss": 1.5274, + "theoretical_loss": 3.8928112222016455, + "tokens_seen": 527040512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042437311935807423, + "loss": 1.4975, + "theoretical_loss": 3.8927079072386186, + "tokens_seen": 527171584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243530591775326, + "loss": 1.3574, + "theoretical_loss": 3.8926046251503092, + "tokens_seen": 527302656 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042433299899699095, + "loss": 1.4539, + "theoretical_loss": 3.892501375918089, + "tokens_seen": 527433728 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3826979398727417, + "objective/train/avg_token_score": 0.035878412425518036, + "objective/train/docs_used": 309026, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3176822662353516, + "objective/train/original_loss": 4.551613807678223, + "objective/train/theoretical_loss": 3.892398159523345, + "objective/train/tokens_used": 527564800, + "objective/train/unlikelihood_loss": 0.1040167510509491, + "theoretical_loss": 3.892398159523345, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243129388164493, + "loss": 1.4981, + "theoretical_loss": 3.892398159523345, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042429287863590773, + "loss": 1.5112, + "theoretical_loss": 3.892294975947477, + "tokens_seen": 527695872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242728184553661, + "loss": 1.4593, + "theoretical_loss": 3.8921918251719054, + "tokens_seen": 527826944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042425275827482446, + "loss": 1.6896, + "theoretical_loss": 3.8920887071780594, + "tokens_seen": 527958016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242326980942828, + "loss": 1.4538, + "theoretical_loss": 3.891985621947388, + "tokens_seen": 528089088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242126379137412, + "loss": 1.5069, + "theoretical_loss": 3.891882569461353, + "tokens_seen": 528220160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241925777331996, + "loss": 1.6161, + "theoretical_loss": 3.8917795497014316, + "tokens_seen": 528351232 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424172517552658, + "loss": 1.315, + "theoretical_loss": 3.891676562649116, + "tokens_seen": 528482304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241524573721164, + "loss": 1.4037, + "theoretical_loss": 3.891573608285915, + "tokens_seen": 528613376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042413239719157474, + "loss": 1.5572, + "theoretical_loss": 3.89147068659335, + "tokens_seen": 528744448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042411233701103316, + "loss": 1.5609, + "theoretical_loss": 3.891367797552959, + "tokens_seen": 528875520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240922768304915, + "loss": 1.5477, + "theoretical_loss": 3.891264941146295, + "tokens_seen": 529006592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240722166499499, + "loss": 1.4536, + "theoretical_loss": 3.8911621173549253, + "tokens_seen": 529137664 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.5902981162071228, + "objective/train/avg_token_score": 0.02185690961778164, + "objective/train/docs_used": 310459, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0812337398529053, + "objective/train/original_loss": 3.739985227584839, + "objective/train/theoretical_loss": 3.89111071768422, + "objective/train/tokens_used": 529203200, + "objective/train/unlikelihood_loss": 0.1313895732164383, + "theoretical_loss": 3.89111071768422, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042405215646940824, + "loss": 1.5481, + "theoretical_loss": 3.8910593261604323, + "tokens_seen": 529268736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240320962888666, + "loss": 1.3968, + "theoretical_loss": 3.8909565675444138, + "tokens_seen": 529399808 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424012036108325, + "loss": 1.4023, + "theoretical_loss": 3.8908538414884823, + "tokens_seen": 529530880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239919759277834, + "loss": 1.3563, + "theoretical_loss": 3.890751147974265, + "tokens_seen": 529661952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042397191574724175, + "loss": 1.6032, + "theoretical_loss": 3.890648486983405, + "tokens_seen": 529793024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239518555667001, + "loss": 1.5371, + "theoretical_loss": 3.8905458584975583, + "tokens_seen": 529924096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239317953861585, + "loss": 1.5931, + "theoretical_loss": 3.8904432624983976, + "tokens_seen": 530055168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239117352056169, + "loss": 1.5401, + "theoretical_loss": 3.89034069896761, + "tokens_seen": 530186240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042389167502507525, + "loss": 1.4977, + "theoretical_loss": 3.8902381678868974, + "tokens_seen": 530317312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238716148445336, + "loss": 1.4138, + "theoretical_loss": 3.890135669237976, + "tokens_seen": 530448384 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423851554663992, + "loss": 1.4784, + "theoretical_loss": 3.8900332030025773, + "tokens_seen": 530579456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238314944834504, + "loss": 1.3966, + "theoretical_loss": 3.889930769162448, + "tokens_seen": 530710528 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.38929617404937744, + "objective/train/avg_token_score": 0.033991601318120956, + "objective/train/docs_used": 311126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3811792135238647, + "objective/train/original_loss": 4.070007801055908, + "objective/train/theoretical_loss": 3.889828367699349, + "objective/train/tokens_used": 530841600, + "objective/train/unlikelihood_loss": 0.13424085080623627, + "theoretical_loss": 3.889828367699349, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042381143430290875, + "loss": 1.4535, + "theoretical_loss": 3.889828367699349, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237913741223671, + "loss": 1.4853, + "theoretical_loss": 3.8897259985950554, + "tokens_seen": 530972672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237713139418255, + "loss": 1.5492, + "theoretical_loss": 3.889623661831359, + "tokens_seen": 531103744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237512537612839, + "loss": 1.432, + "theoretical_loss": 3.8895213573900635, + "tokens_seen": 531234816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042373119358074226, + "loss": 1.404, + "theoretical_loss": 3.8894190852529906, + "tokens_seen": 531365888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237111334002006, + "loss": 1.3642, + "theoretical_loss": 3.889316845401974, + "tokens_seen": 531496960 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423691073219659, + "loss": 1.4066, + "theoretical_loss": 3.8892146378188643, + "tokens_seen": 531628032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236710130391174, + "loss": 1.576, + "theoretical_loss": 3.8891124624855236, + "tokens_seen": 531759104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042365095285857576, + "loss": 1.4427, + "theoretical_loss": 3.889010319383832, + "tokens_seen": 531890176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236308926780341, + "loss": 1.3313, + "theoretical_loss": 3.888908208495683, + "tokens_seen": 532021248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236108324974925, + "loss": 1.5414, + "theoretical_loss": 3.888806129802984, + "tokens_seen": 532152320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042359077231695085, + "loss": 1.5081, + "theoretical_loss": 3.8887040832876587, + "tokens_seen": 532283392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042357071213640926, + "loss": 1.5332, + "theoretical_loss": 3.888602068931643, + "tokens_seen": 532414464 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3950391113758087, + "objective/train/avg_token_score": 0.011321122758090496, + "objective/train/docs_used": 312276, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4738465547561646, + "objective/train/original_loss": 3.9406888484954834, + "objective/train/theoretical_loss": 3.888551073807736, + "objective/train/tokens_used": 532480000, + "objective/train/unlikelihood_loss": 0.1509210616350174, + "theoretical_loss": 3.888551073807736, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235506519558676, + "loss": 1.4874, + "theoretical_loss": 3.8885000867168893, + "tokens_seen": 532545536 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423530591775326, + "loss": 1.4809, + "theoretical_loss": 3.888398136625364, + "tokens_seen": 532676608 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042351053159478435, + "loss": 1.4155, + "theoretical_loss": 3.8882962186390486, + "tokens_seen": 532807680 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042349047141424276, + "loss": 1.4746, + "theoretical_loss": 3.8881943327399378, + "tokens_seen": 532938752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042347041123370113, + "loss": 1.5315, + "theoretical_loss": 3.8880924789100417, + "tokens_seen": 533069824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234503510531595, + "loss": 1.4988, + "theoretical_loss": 3.8879906571313847, + "tokens_seen": 533200896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042343029087261785, + "loss": 1.4565, + "theoretical_loss": 3.887888867386007, + "tokens_seen": 533331968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234102306920762, + "loss": 1.4381, + "theoretical_loss": 3.88778710965596, + "tokens_seen": 533463040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042339017051153463, + "loss": 1.4021, + "theoretical_loss": 3.887685383923314, + "tokens_seen": 533594112 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423370110330993, + "loss": 1.5644, + "theoretical_loss": 3.887583690170149, + "tokens_seen": 533725184 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042335005015045135, + "loss": 1.5509, + "theoretical_loss": 3.887482028378564, + "tokens_seen": 533856256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233299899699097, + "loss": 1.4937, + "theoretical_loss": 3.8873803985306683, + "tokens_seen": 533987328 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.49890029430389404, + "objective/train/avg_token_score": 0.006122749298810959, + "objective/train/docs_used": 312792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6340548992156982, + "objective/train/original_loss": 3.637239456176758, + "objective/train/theoretical_loss": 3.8872788006085894, + "objective/train/tokens_used": 534118400, + "objective/train/unlikelihood_loss": 0.22649259865283966, + "theoretical_loss": 3.8872788006085894, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042330992978936813, + "loss": 1.5323, + "theoretical_loss": 3.8872788006085894, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232898696088265, + "loss": 1.3563, + "theoretical_loss": 3.887177234594466, + "tokens_seen": 534249472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042326980942828486, + "loss": 1.2005, + "theoretical_loss": 3.8870757004704526, + "tokens_seen": 534380544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232497492477432, + "loss": 1.5123, + "theoretical_loss": 3.8869741982187183, + "tokens_seen": 534511616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232296890672016, + "loss": 1.5522, + "theoretical_loss": 3.886872727821446, + "tokens_seen": 534642688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042320962888666, + "loss": 1.5628, + "theoretical_loss": 3.886771289260834, + "tokens_seen": 534773760 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042318956870611836, + "loss": 1.6368, + "theoretical_loss": 3.886669882519093, + "tokens_seen": 534904832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231695085255767, + "loss": 1.6142, + "theoretical_loss": 3.886568507578448, + "tokens_seen": 535035904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231494483450351, + "loss": 1.4894, + "theoretical_loss": 3.886467164421142, + "tokens_seen": 535166976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231293881644935, + "loss": 1.5009, + "theoretical_loss": 3.886365853029427, + "tokens_seen": 535298048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042310932798395186, + "loss": 1.3925, + "theoretical_loss": 3.886264573385574, + "tokens_seen": 535429120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230892678034102, + "loss": 1.4485, + "theoretical_loss": 3.886163325471864, + "tokens_seen": 535560192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230692076228686, + "loss": 1.4131, + "theoretical_loss": 3.8860621092705956, + "tokens_seen": 535691264 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4310850501060486, + "objective/train/avg_token_score": 0.006272444501519203, + "objective/train/docs_used": 314102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.511539101600647, + "objective/train/original_loss": 3.6116085052490234, + "objective/train/theoretical_loss": 3.8860115130565984, + "objective/train/tokens_used": 535756800, + "objective/train/unlikelihood_loss": 0.3091060221195221, + "theoretical_loss": 3.8860115130565984, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042304914744232695, + "loss": 1.4495, + "theoretical_loss": 3.8859609247640794, + "tokens_seen": 535822336 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042302908726178537, + "loss": 1.4349, + "theoretical_loss": 3.885859771934642, + "tokens_seen": 535953408 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042300902708124373, + "loss": 1.1708, + "theoretical_loss": 3.8857586507646227, + "tokens_seen": 536084480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229889669007021, + "loss": 1.4465, + "theoretical_loss": 3.8856575612363753, + "tokens_seen": 536215552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042296890672016045, + "loss": 1.5385, + "theoretical_loss": 3.885556503332268, + "tokens_seen": 536346624 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042294884653961887, + "loss": 1.5361, + "theoretical_loss": 3.885455477034683, + "tokens_seen": 536477696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042292878635907723, + "loss": 1.4312, + "theoretical_loss": 3.8853544823260178, + "tokens_seen": 536608768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229087261785356, + "loss": 1.4874, + "theoretical_loss": 3.885253519188682, + "tokens_seen": 536739840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042288866599799396, + "loss": 1.431, + "theoretical_loss": 3.8851525876050994, + "tokens_seen": 536870912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228686058174523, + "loss": 1.3144, + "theoretical_loss": 3.8850516875577092, + "tokens_seen": 537001984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284854563691074, + "loss": 1.4203, + "theoretical_loss": 3.884950819028965, + "tokens_seen": 537133056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228284854563691, + "loss": 1.5352, + "theoretical_loss": 3.8848499820013327, + "tokens_seen": 537264128 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.45100194215774536, + "objective/train/avg_token_score": 0.006222711876034737, + "objective/train/docs_used": 314596, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7215205430984497, + "objective/train/original_loss": 3.9992692470550537, + "objective/train/theoretical_loss": 3.8847491764572926, + "objective/train/tokens_used": 537395200, + "objective/train/unlikelihood_loss": 0.1294986605644226, + "theoretical_loss": 3.8847491764572926, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042280842527582746, + "loss": 1.4693, + "theoretical_loss": 3.8847491764572926, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227883650952858, + "loss": 1.3508, + "theoretical_loss": 3.884648402379341, + "tokens_seen": 537526272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042276830491474424, + "loss": 1.4451, + "theoretical_loss": 3.8845476597499857, + "tokens_seen": 537657344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227482447342026, + "loss": 1.41, + "theoretical_loss": 3.8844469485517488, + "tokens_seen": 537788416 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042272818455366096, + "loss": 1.4082, + "theoretical_loss": 3.8843462687671684, + "tokens_seen": 537919488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227081243731193, + "loss": 1.3873, + "theoretical_loss": 3.884245620378794, + "tokens_seen": 538050560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226880641925777, + "loss": 1.5178, + "theoretical_loss": 3.8841450033691913, + "tokens_seen": 538181632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042266800401203616, + "loss": 1.4785, + "theoretical_loss": 3.884044417720938, + "tokens_seen": 538312704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226479438314945, + "loss": 1.527, + "theoretical_loss": 3.8839438634166275, + "tokens_seen": 538443776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226278836509529, + "loss": 1.4577, + "theoretical_loss": 3.883843340438865, + "tokens_seen": 538574848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042260782347041125, + "loss": 1.3839, + "theoretical_loss": 3.8837428487702716, + "tokens_seen": 538705920 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042258776328986966, + "loss": 1.5392, + "theoretical_loss": 3.883642388393481, + "tokens_seen": 538836992 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422567703109328, + "loss": 1.5504, + "theoretical_loss": 3.8835419592911413, + "tokens_seen": 538968064 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.32624632120132446, + "objective/train/avg_token_score": 0.007150240708142519, + "objective/train/docs_used": 315427, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1273283958435059, + "objective/train/original_loss": 3.809842348098755, + "objective/train/theoretical_loss": 3.8834917564624716, + "objective/train/tokens_used": 539033600, + "objective/train/unlikelihood_loss": 0.1816972941160202, + "theoretical_loss": 3.8834917564624716, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225476429287864, + "loss": 1.3572, + "theoretical_loss": 3.883441561445914, + "tokens_seen": 539099136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252758274824475, + "loss": 1.5028, + "theoretical_loss": 3.8833411948404755, + "tokens_seen": 539230208 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042250752256770317, + "loss": 1.3948, + "theoretical_loss": 3.8832408594575147, + "tokens_seen": 539361280 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248746238716153, + "loss": 1.2123, + "theoretical_loss": 3.883140555279735, + "tokens_seen": 539492352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224674022066199, + "loss": 1.3314, + "theoretical_loss": 3.8830402822898535, + "tokens_seen": 539623424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042244734202607825, + "loss": 1.4584, + "theoretical_loss": 3.8829400404706003, + "tokens_seen": 539754496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224272818455366, + "loss": 1.4907, + "theoretical_loss": 3.8828398298047206, + "tokens_seen": 539885568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042240722166499503, + "loss": 1.5874, + "theoretical_loss": 3.8827396502749734, + "tokens_seen": 540016640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223871614844534, + "loss": 1.4599, + "theoretical_loss": 3.882639501864129, + "tokens_seen": 540147712 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042236710130391176, + "loss": 1.3443, + "theoretical_loss": 3.882539384554974, + "tokens_seen": 540278784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223470411233701, + "loss": 1.4215, + "theoretical_loss": 3.8824392983303078, + "tokens_seen": 540409856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042232698094282853, + "loss": 1.5224, + "theoretical_loss": 3.8823392431729435, + "tokens_seen": 540540928 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3801930546760559, + "objective/train/avg_token_score": 0.03133194521069527, + "objective/train/docs_used": 316530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4709320068359375, + "objective/train/original_loss": 4.609292030334473, + "objective/train/theoretical_loss": 3.882239219065708, + "objective/train/tokens_used": 540672000, + "objective/train/unlikelihood_loss": 0.10041511058807373, + "theoretical_loss": 3.882239219065708, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223069207622869, + "loss": 1.531, + "theoretical_loss": 3.882239219065708, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042228686058174526, + "loss": 1.3437, + "theoretical_loss": 3.882139225991441, + "tokens_seen": 540803072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222668004012036, + "loss": 1.6213, + "theoretical_loss": 3.882039263932997, + "tokens_seen": 540934144 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422246740220662, + "loss": 1.3605, + "theoretical_loss": 3.8819393328732446, + "tokens_seen": 541065216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222266800401204, + "loss": 1.459, + "theoretical_loss": 3.881839432795064, + "tokens_seen": 541196288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042220661985957876, + "loss": 1.525, + "theoretical_loss": 3.88173956368135, + "tokens_seen": 541327360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221865596790371, + "loss": 1.541, + "theoretical_loss": 3.8816397255150115, + "tokens_seen": 541458432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221664994984955, + "loss": 1.2984, + "theoretical_loss": 3.8815399182789703, + "tokens_seen": 541589504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221464393179539, + "loss": 1.4195, + "theoretical_loss": 3.881440141956162, + "tokens_seen": 541720576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042212637913741226, + "loss": 1.3955, + "theoretical_loss": 3.8813403965295357, + "tokens_seen": 541851648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221063189568706, + "loss": 1.4605, + "theoretical_loss": 3.881240681982054, + "tokens_seen": 541982720 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422086258776329, + "loss": 1.4174, + "theoretical_loss": 3.881140998296694, + "tokens_seen": 542113792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042206619859578735, + "loss": 1.4816, + "theoretical_loss": 3.881041345456444, + "tokens_seen": 542244864 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.44330400228500366, + "objective/train/avg_token_score": 0.020557841286063194, + "objective/train/docs_used": 317270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5593016147613525, + "objective/train/original_loss": 3.649369955062866, + "objective/train/theoretical_loss": 3.8809915305979237, + "objective/train/tokens_used": 542310400, + "objective/train/unlikelihood_loss": 0.21089822053909302, + "theoretical_loss": 3.8809915305979237, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042204613841524577, + "loss": 1.4163, + "theoretical_loss": 3.880941723444308, + "tokens_seen": 542375936 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042202607823470413, + "loss": 1.5283, + "theoretical_loss": 3.8808421322433024, + "tokens_seen": 542507008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220060180541625, + "loss": 1.469, + "theoretical_loss": 3.880742571836457, + "tokens_seen": 542638080 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042198595787362085, + "loss": 1.3293, + "theoretical_loss": 3.880643042206816, + "tokens_seen": 542769152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042196589769307927, + "loss": 1.4443, + "theoretical_loss": 3.8805435433374353, + "tokens_seen": 542900224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042194583751253763, + "loss": 1.4672, + "theoretical_loss": 3.8804440752113862, + "tokens_seen": 543031296 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421925777331996, + "loss": 1.4297, + "theoretical_loss": 3.880344637811752, + "tokens_seen": 543162368 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042190571715145436, + "loss": 1.4319, + "theoretical_loss": 3.8802452311216302, + "tokens_seen": 543293440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218856569709127, + "loss": 1.5163, + "theoretical_loss": 3.8801458551241312, + "tokens_seen": 543424512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042186559679037114, + "loss": 1.471, + "theoretical_loss": 3.880046509802378, + "tokens_seen": 543555584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218455366098295, + "loss": 1.4938, + "theoretical_loss": 3.8799471951395095, + "tokens_seen": 543686656 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042182547642928786, + "loss": 1.5712, + "theoretical_loss": 3.8798479111186754, + "tokens_seen": 543817728 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3662634491920471, + "objective/train/avg_token_score": 0.021842803806066513, + "objective/train/docs_used": 318285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.198555588722229, + "objective/train/original_loss": 3.5977704524993896, + "objective/train/theoretical_loss": 3.879748657723039, + "objective/train/tokens_used": 543948800, + "objective/train/unlikelihood_loss": 0.24941086769104004, + "theoretical_loss": 3.879748657723039, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218054162487462, + "loss": 1.4546, + "theoretical_loss": 3.879748657723039, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042178535606820464, + "loss": 1.4712, + "theoretical_loss": 3.879649434935778, + "tokens_seen": 544079872 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421765295887663, + "loss": 1.6343, + "theoretical_loss": 3.879550242740083, + "tokens_seen": 544210944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042174523570712136, + "loss": 1.5444, + "theoretical_loss": 3.879451081119158, + "tokens_seen": 544342016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217251755265797, + "loss": 1.6409, + "theoretical_loss": 3.8793519500562192, + "tokens_seen": 544473088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004217051153460381, + "loss": 1.521, + "theoretical_loss": 3.879252849534498, + "tokens_seen": 544604160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216850551654965, + "loss": 1.4057, + "theoretical_loss": 3.8791537795372366, + "tokens_seen": 544735232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042166499498495487, + "loss": 1.6129, + "theoretical_loss": 3.8790547400476925, + "tokens_seen": 544866304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042164493480441323, + "loss": 1.5271, + "theoretical_loss": 3.878955731049136, + "tokens_seen": 544997376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216248746238716, + "loss": 1.496, + "theoretical_loss": 3.8788567525248494, + "tokens_seen": 545128448 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042160481444333, + "loss": 1.3628, + "theoretical_loss": 3.8787578044581292, + "tokens_seen": 545259520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042158475426278837, + "loss": 1.4522, + "theoretical_loss": 3.8786588868322847, + "tokens_seen": 545390592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042156469408224673, + "loss": 1.3902, + "theoretical_loss": 3.8785599996306397, + "tokens_seen": 545521664 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.4580278694629669, + "objective/train/avg_token_score": 0.007631353102624416, + "objective/train/docs_used": 318961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5138821601867676, + "objective/train/original_loss": 3.7409567832946777, + "objective/train/theoretical_loss": 3.8785105674336835, + "objective/train/tokens_used": 545587200, + "objective/train/unlikelihood_loss": 0.14674793183803558, + "theoretical_loss": 3.8785105674336835, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215446339017051, + "loss": 1.4147, + "theoretical_loss": 3.8784611428365285, + "tokens_seen": 545652736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215245737211635, + "loss": 1.4939, + "theoretical_loss": 3.878362316433301, + "tokens_seen": 545783808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215045135406219, + "loss": 1.4692, + "theoretical_loss": 3.8782635204043188, + "tokens_seen": 545914880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042148445336008024, + "loss": 1.4711, + "theoretical_loss": 3.8781647547329565, + "tokens_seen": 546045952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214643931795386, + "loss": 1.5982, + "theoretical_loss": 3.8780660194026035, + "tokens_seen": 546177024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042144433299899696, + "loss": 1.5872, + "theoretical_loss": 3.8779673143966606, + "tokens_seen": 546308096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214242728184554, + "loss": 1.6839, + "theoretical_loss": 3.8778686396985416, + "tokens_seen": 546439168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042140421263791374, + "loss": 1.4686, + "theoretical_loss": 3.8777699952916738, + "tokens_seen": 546570240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213841524573721, + "loss": 1.6047, + "theoretical_loss": 3.877671381159499, + "tokens_seen": 546701312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042136409227683046, + "loss": 1.331, + "theoretical_loss": 3.877572797285469, + "tokens_seen": 546832384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213440320962889, + "loss": 1.3915, + "theoretical_loss": 3.877474243653051, + "tokens_seen": 546963456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042132397191574724, + "loss": 1.5564, + "theoretical_loss": 3.877375720245725, + "tokens_seen": 547094528 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.41086265444755554, + "objective/train/avg_token_score": 0.00896799098700285, + "objective/train/docs_used": 320353, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4873528480529785, + "objective/train/original_loss": 3.9619879722595215, + "objective/train/theoretical_loss": 3.8772772270469824, + "objective/train/tokens_used": 547225600, + "objective/train/unlikelihood_loss": 0.1763414591550827, + "theoretical_loss": 3.8772772270469824, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213039117352056, + "loss": 1.5364, + "theoretical_loss": 3.8772772270469824, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042128385155466397, + "loss": 1.4582, + "theoretical_loss": 3.877178764040329, + "tokens_seen": 547356672 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042126379137412233, + "loss": 1.5835, + "theoretical_loss": 3.8770803312092834, + "tokens_seen": 547487744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042124373119358075, + "loss": 1.3389, + "theoretical_loss": 3.8769819285373766, + "tokens_seen": 547618816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212236710130391, + "loss": 1.518, + "theoretical_loss": 3.8768835560081527, + "tokens_seen": 547749888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042120361083249747, + "loss": 1.417, + "theoretical_loss": 3.8767852136051686, + "tokens_seen": 547880960 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042118355065195583, + "loss": 1.46, + "theoretical_loss": 3.876686901311995, + "tokens_seen": 548012032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211634904714143, + "loss": 1.6196, + "theoretical_loss": 3.8765886191122148, + "tokens_seen": 548143104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042114343029087266, + "loss": 1.4089, + "theoretical_loss": 3.876490366989423, + "tokens_seen": 548274176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042112337011033103, + "loss": 1.4457, + "theoretical_loss": 3.876392144927229, + "tokens_seen": 548405248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211033099297894, + "loss": 1.4709, + "theoretical_loss": 3.8762939529092537, + "tokens_seen": 548536320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042108324974924775, + "loss": 1.6226, + "theoretical_loss": 3.876195790919132, + "tokens_seen": 548667392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042106318956870617, + "loss": 1.5583, + "theoretical_loss": 3.8760976589405107, + "tokens_seen": 548798464 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.4123900234699249, + "objective/train/avg_token_score": 0.014366233721375465, + "objective/train/docs_used": 320989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.500076174736023, + "objective/train/original_loss": 3.8920931816101074, + "objective/train/theoretical_loss": 3.8760486042004056, + "objective/train/tokens_used": 548864000, + "objective/train/unlikelihood_loss": 0.15627138316631317, + "theoretical_loss": 3.8760486042004056, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042104312938816453, + "loss": 1.426, + "theoretical_loss": 3.8759995569570496, + "tokens_seen": 548929536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210230692076229, + "loss": 1.4949, + "theoretical_loss": 3.875901484952422, + "tokens_seen": 549060608 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042100300902708125, + "loss": 1.5319, + "theoretical_loss": 3.8758034429103128, + "tokens_seen": 549191680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042098294884653967, + "loss": 1.6009, + "theoretical_loss": 3.875705430814421, + "tokens_seen": 549322752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042096288866599803, + "loss": 1.2854, + "theoretical_loss": 3.8756074486484566, + "tokens_seen": 549453824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209428284854564, + "loss": 1.3986, + "theoretical_loss": 3.8755094963961447, + "tokens_seen": 549584896 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042092276830491476, + "loss": 1.4487, + "theoretical_loss": 3.8754115740412214, + "tokens_seen": 549715968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209027081243731, + "loss": 1.4448, + "theoretical_loss": 3.8753136815674356, + "tokens_seen": 549847040 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042088264794383154, + "loss": 1.5002, + "theoretical_loss": 3.875215818958549, + "tokens_seen": 549978112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208625877632899, + "loss": 1.6046, + "theoretical_loss": 3.8751179861983367, + "tokens_seen": 550109184 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042084252758274826, + "loss": 1.3805, + "theoretical_loss": 3.8750201832705864, + "tokens_seen": 550240256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208224674022066, + "loss": 1.6252, + "theoretical_loss": 3.8749224101590976, + "tokens_seen": 550371328 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.3899071216583252, + "objective/train/avg_token_score": 0.00873059593141079, + "objective/train/docs_used": 322374, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3493757247924805, + "objective/train/original_loss": 3.7656335830688477, + "objective/train/theoretical_loss": 3.8748246668476827, + "objective/train/tokens_used": 550502400, + "objective/train/unlikelihood_loss": 0.2136422097682953, + "theoretical_loss": 3.8748246668476827, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042080240722166504, + "loss": 1.5383, + "theoretical_loss": 3.8748246668476827, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207823470411234, + "loss": 1.4345, + "theoretical_loss": 3.8747269533201676, + "tokens_seen": 550633472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042076228686058176, + "loss": 1.4031, + "theoretical_loss": 3.8746292695603897, + "tokens_seen": 550764544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207422266800401, + "loss": 1.6027, + "theoretical_loss": 3.8745316155521996, + "tokens_seen": 550895616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207221664994985, + "loss": 1.6353, + "theoretical_loss": 3.874433991279461, + "tokens_seen": 551026688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207021063189569, + "loss": 1.5923, + "theoretical_loss": 3.874336396726049, + "tokens_seen": 551157760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042068204613841527, + "loss": 1.5882, + "theoretical_loss": 3.874238831875852, + "tokens_seen": 551288832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042066198595787363, + "loss": 1.5571, + "theoretical_loss": 3.8741412967127706, + "tokens_seen": 551419904 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420641925777332, + "loss": 1.3437, + "theoretical_loss": 3.8740437912207186, + "tokens_seen": 551550976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206218655967904, + "loss": 1.463, + "theoretical_loss": 3.873946315383622, + "tokens_seen": 551682048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042060180541624877, + "loss": 1.4278, + "theoretical_loss": 3.8738488691854194, + "tokens_seen": 551813120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042058174523570713, + "loss": 1.2151, + "theoretical_loss": 3.873751452610061, + "tokens_seen": 551944192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205616850551655, + "loss": 1.4381, + "theoretical_loss": 3.873654065641511, + "tokens_seen": 552075264 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.48796433210372925, + "objective/train/avg_token_score": 0.016625795513391495, + "objective/train/docs_used": 322945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7378815412521362, + "objective/train/original_loss": 4.003293037414551, + "objective/train/theoretical_loss": 3.8736053832547808, + "objective/train/tokens_used": 552140800, + "objective/train/unlikelihood_loss": 0.14501631259918213, + "theoretical_loss": 3.8736053832547808, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205416248746239, + "loss": 1.4769, + "theoretical_loss": 3.873556708263745, + "tokens_seen": 552206336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205215646940823, + "loss": 1.4547, + "theoretical_loss": 3.8734593804607513, + "tokens_seen": 552337408 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042050150451354064, + "loss": 1.5517, + "theoretical_loss": 3.8733620822165316, + "tokens_seen": 552468480 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420481444332999, + "loss": 1.4435, + "theoretical_loss": 3.8732648135150978, + "tokens_seen": 552599552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042046138415245736, + "loss": 1.565, + "theoretical_loss": 3.873167574340477, + "tokens_seen": 552730624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204413239719158, + "loss": 1.3768, + "theoretical_loss": 3.873070364676707, + "tokens_seen": 552861696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042042126379137414, + "loss": 1.5248, + "theoretical_loss": 3.8729731845078383, + "tokens_seen": 552992768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204012036108325, + "loss": 1.1907, + "theoretical_loss": 3.872876033817934, + "tokens_seen": 553123840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042038114343029086, + "loss": 1.432, + "theoretical_loss": 3.872778912591069, + "tokens_seen": 553254912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203610832497493, + "loss": 1.4321, + "theoretical_loss": 3.8726818208113323, + "tokens_seen": 553385984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042034102306920764, + "loss": 1.3276, + "theoretical_loss": 3.872584758462823, + "tokens_seen": 553517056 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420320962888666, + "loss": 1.6415, + "theoretical_loss": 3.8724877255296533, + "tokens_seen": 553648128 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.4739125072956085, + "objective/train/avg_token_score": 0.016837503761053085, + "objective/train/docs_used": 324208, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4981387853622437, + "objective/train/original_loss": 3.6097116470336914, + "objective/train/theoretical_loss": 3.8723907219959486, + "objective/train/tokens_used": 553779200, + "objective/train/unlikelihood_loss": 0.127224862575531, + "theoretical_loss": 3.8723907219959486, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042030090270812437, + "loss": 1.446, + "theoretical_loss": 3.8723907219959486, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042028084252758273, + "loss": 1.4663, + "theoretical_loss": 3.872293747845846, + "tokens_seen": 553910272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042026078234704115, + "loss": 1.4597, + "theoretical_loss": 3.872196803063495, + "tokens_seen": 554041344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202407221664995, + "loss": 1.5731, + "theoretical_loss": 3.8720998876330572, + "tokens_seen": 554172416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042022066198595787, + "loss": 1.4404, + "theoretical_loss": 3.8720030015387064, + "tokens_seen": 554303488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042020060180541623, + "loss": 1.4862, + "theoretical_loss": 3.8719061447646297, + "tokens_seen": 554434560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042018054162487465, + "loss": 1.2866, + "theoretical_loss": 3.8718093172950248, + "tokens_seen": 554565632 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420160481444333, + "loss": 1.4176, + "theoretical_loss": 3.8717125191141024, + "tokens_seen": 554696704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201404212637914, + "loss": 1.2709, + "theoretical_loss": 3.8716157502060864, + "tokens_seen": 554827776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042012036108324974, + "loss": 1.4054, + "theoretical_loss": 3.871519010555211, + "tokens_seen": 554958848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201003009027081, + "loss": 1.329, + "theoretical_loss": 3.871422300145725, + "tokens_seen": 555089920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200802407221665, + "loss": 1.2811, + "theoretical_loss": 3.871325618961887, + "tokens_seen": 555220992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200601805416249, + "loss": 1.4898, + "theoretical_loss": 3.871228966987969, + "tokens_seen": 555352064 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.40713587403297424, + "objective/train/avg_token_score": 0.009826954454183578, + "objective/train/docs_used": 324930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.531923532485962, + "objective/train/original_loss": 3.965177297592163, + "objective/train/theoretical_loss": 3.871180651949819, + "objective/train/tokens_used": 555417600, + "objective/train/unlikelihood_loss": 0.14890779554843903, + "theoretical_loss": 3.871180651949819, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042004012036108324, + "loss": 1.4487, + "theoretical_loss": 3.871132344208256, + "tokens_seen": 555483136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200200601805416, + "loss": 1.3508, + "theoretical_loss": 3.8710357506070427, + "tokens_seen": 555614208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042, + "loss": 1.5513, + "theoretical_loss": 3.870939186168638, + "tokens_seen": 555745280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199799398194584, + "loss": 1.314, + "theoretical_loss": 3.8708426508773632, + "tokens_seen": 555876352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041995987963891674, + "loss": 1.319, + "theoretical_loss": 3.8707461447175495, + "tokens_seen": 556007424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199398194583751, + "loss": 1.4497, + "theoretical_loss": 3.870649667673543, + "tokens_seen": 556138496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041991975927783347, + "loss": 1.3447, + "theoretical_loss": 3.8705532197296995, + "tokens_seen": 556269568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198996990972919, + "loss": 1.375, + "theoretical_loss": 3.870456800870388, + "tokens_seen": 556400640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041987963891675025, + "loss": 1.4515, + "theoretical_loss": 3.8703604110799903, + "tokens_seen": 556531712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198595787362086, + "loss": 1.3271, + "theoretical_loss": 3.8702640503428984, + "tokens_seen": 556662784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041983951855566697, + "loss": 1.3552, + "theoretical_loss": 3.8701677186435184, + "tokens_seen": 556793856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198194583751254, + "loss": 1.4606, + "theoretical_loss": 3.8700714159662666, + "tokens_seen": 556924928 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.32086998224258423, + "objective/train/avg_token_score": 0.02666344866156578, + "objective/train/docs_used": 326252, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0482410192489624, + "objective/train/original_loss": 4.232210636138916, + "objective/train/theoretical_loss": 3.869975142295573, + "objective/train/tokens_used": 557056000, + "objective/train/unlikelihood_loss": 0.14249522984027863, + "theoretical_loss": 3.869975142295573, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041979939819458375, + "loss": 1.5389, + "theoretical_loss": 3.869975142295573, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197793380140421, + "loss": 1.3538, + "theoretical_loss": 3.8698788976158776, + "tokens_seen": 557187072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041975927783350047, + "loss": 1.315, + "theoretical_loss": 3.8697826819116345, + "tokens_seen": 557318144 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041973921765295883, + "loss": 1.6038, + "theoretical_loss": 3.8696864951673087, + "tokens_seen": 557449216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041971915747241725, + "loss": 1.4843, + "theoretical_loss": 3.8695903373673772, + "tokens_seen": 557580288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196990972918756, + "loss": 1.5741, + "theoretical_loss": 3.869494208496329, + "tokens_seen": 557711360 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419679037111334, + "loss": 1.5376, + "theoretical_loss": 3.8693981085386655, + "tokens_seen": 557842432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196589769307924, + "loss": 1.4148, + "theoretical_loss": 3.8693020374788993, + "tokens_seen": 557973504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196389167502508, + "loss": 1.5087, + "theoretical_loss": 3.8692059953015563, + "tokens_seen": 558104576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041961885656970917, + "loss": 1.2978, + "theoretical_loss": 3.8691099819911723, + "tokens_seen": 558235648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041959879638916753, + "loss": 1.468, + "theoretical_loss": 3.8690139975322966, + "tokens_seen": 558366720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195787362086259, + "loss": 1.275, + "theoretical_loss": 3.8689180419094895, + "tokens_seen": 558497792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195586760280843, + "loss": 1.5295, + "theoretical_loss": 3.868822115107324, + "tokens_seen": 558628864 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.48692572116851807, + "objective/train/avg_token_score": 0.008406232111155987, + "objective/train/docs_used": 326886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.637534499168396, + "objective/train/original_loss": 3.788033962249756, + "objective/train/theoretical_loss": 3.8687741625091636, + "objective/train/tokens_used": 558694400, + "objective/train/unlikelihood_loss": 0.1396278291940689, + "theoretical_loss": 3.8687741625091636, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195386158475427, + "loss": 1.4419, + "theoretical_loss": 3.8687262171103836, + "tokens_seen": 558759936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041951855566700104, + "loss": 1.6244, + "theoretical_loss": 3.868630347903266, + "tokens_seen": 558891008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194984954864594, + "loss": 1.3994, + "theoretical_loss": 3.868534507470579, + "tokens_seen": 559022080 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041947843530591776, + "loss": 1.4045, + "theoretical_loss": 3.868438695796941, + "tokens_seen": 559153152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194583751253762, + "loss": 1.3849, + "theoretical_loss": 3.8683429128669857, + "tokens_seen": 559284224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943831494483454, + "loss": 1.3363, + "theoretical_loss": 3.868247158665356, + "tokens_seen": 559415296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194182547642929, + "loss": 1.3883, + "theoretical_loss": 3.868151433176707, + "tokens_seen": 559546368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041939819458375126, + "loss": 1.5001, + "theoretical_loss": 3.868055736385706, + "tokens_seen": 559677440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193781344032097, + "loss": 1.4969, + "theoretical_loss": 3.867960068277033, + "tokens_seen": 559808512 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041935807422266804, + "loss": 1.3821, + "theoretical_loss": 3.867864428835377, + "tokens_seen": 559939584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193380140421264, + "loss": 1.4965, + "theoretical_loss": 3.8677688180454415, + "tokens_seen": 560070656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041931795386158477, + "loss": 1.5585, + "theoretical_loss": 3.8676732358919406, + "tokens_seen": 560201728 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.2959432899951935, + "objective/train/avg_token_score": 0.025653989985585213, + "objective/train/docs_used": 327930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9278034567832947, + "objective/train/original_loss": 3.611701011657715, + "objective/train/theoretical_loss": 3.8675776823595998, + "objective/train/tokens_used": 560332800, + "objective/train/unlikelihood_loss": 0.2902003228664398, + "theoretical_loss": 3.8675776823595998, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041929789368104313, + "loss": 1.3127, + "theoretical_loss": 3.8675776823595998, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927783350050155, + "loss": 1.5624, + "theoretical_loss": 3.867482157433157, + "tokens_seen": 560463872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192577733199599, + "loss": 1.3398, + "theoretical_loss": 3.8673866610973624, + "tokens_seen": 560594944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041923771313941827, + "loss": 1.4846, + "theoretical_loss": 3.867291193336976, + "tokens_seen": 560726016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041921765295887663, + "loss": 1.5351, + "theoretical_loss": 3.867195754136771, + "tokens_seen": 560857088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041919759277833505, + "loss": 1.4848, + "theoretical_loss": 3.867100343481532, + "tokens_seen": 560988160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191775325977934, + "loss": 1.3812, + "theoretical_loss": 3.867004961356054, + "tokens_seen": 561119232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191574724172518, + "loss": 1.3775, + "theoretical_loss": 3.8669096077451464, + "tokens_seen": 561250304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041913741223671014, + "loss": 1.6188, + "theoretical_loss": 3.8668142826336274, + "tokens_seen": 561381376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191173520561685, + "loss": 1.4986, + "theoretical_loss": 3.8667189860063282, + "tokens_seen": 561512448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190972918756269, + "loss": 1.3638, + "theoretical_loss": 3.866623717848092, + "tokens_seen": 561643520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190772316950853, + "loss": 1.4552, + "theoretical_loss": 3.8665284781437714, + "tokens_seen": 561774592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041905717151454364, + "loss": 1.4597, + "theoretical_loss": 3.866433266878234, + "tokens_seen": 561905664 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.5047653913497925, + "objective/train/avg_token_score": 0.00482219597324729, + "objective/train/docs_used": 328746, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.737789511680603, + "objective/train/original_loss": 3.9125659465789795, + "objective/train/theoretical_loss": 3.8663856719052827, + "objective/train/tokens_used": 561971200, + "objective/train/unlikelihood_loss": 0.14458067715168, + "theoretical_loss": 3.8663856719052827, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419037111334002, + "loss": 1.5193, + "theoretical_loss": 3.8663380840363564, + "tokens_seen": 562036736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190170511534604, + "loss": 1.362, + "theoretical_loss": 3.866242929603027, + "tokens_seen": 562167808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189969909729188, + "loss": 1.4048, + "theoretical_loss": 3.866147803563148, + "tokens_seen": 562298880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041897693079237714, + "loss": 1.6211, + "theoretical_loss": 3.8660527059016294, + "tokens_seen": 562429952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189568706118355, + "loss": 1.4695, + "theoretical_loss": 3.8659576366033956, + "tokens_seen": 562561024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041893681043129387, + "loss": 1.4011, + "theoretical_loss": 3.8658625956533816, + "tokens_seen": 562692096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189167502507523, + "loss": 1.3756, + "theoretical_loss": 3.8657675830365346, + "tokens_seen": 562823168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041889669007021065, + "loss": 1.4237, + "theoretical_loss": 3.8656725987378118, + "tokens_seen": 562954240 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418876629889669, + "loss": 1.4832, + "theoretical_loss": 3.865577642742183, + "tokens_seen": 563085312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041885656970912737, + "loss": 1.4726, + "theoretical_loss": 3.86548271503463, + "tokens_seen": 563216384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188365095285858, + "loss": 1.3446, + "theoretical_loss": 3.865387815600144, + "tokens_seen": 563347456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041881644934804415, + "loss": 1.5273, + "theoretical_loss": 3.86529294442373, + "tokens_seen": 563478528 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.6345918774604797, + "objective/train/avg_token_score": 0.00891077984124422, + "objective/train/docs_used": 329425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0925443172454834, + "objective/train/original_loss": 3.6826608180999756, + "objective/train/theoretical_loss": 3.8651981014904027, + "objective/train/tokens_used": 563609600, + "objective/train/unlikelihood_loss": 0.14516757428646088, + "theoretical_loss": 3.8651981014904027, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187963891675025, + "loss": 1.452, + "theoretical_loss": 3.8651981014904027, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187763289869609, + "loss": 1.2785, + "theoretical_loss": 3.8651032867851893, + "tokens_seen": 563740672 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041875626880641924, + "loss": 1.3895, + "theoretical_loss": 3.8650085002931283, + "tokens_seen": 563871744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041873620862587765, + "loss": 1.4778, + "theoretical_loss": 3.8649137419992687, + "tokens_seen": 564002816 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418716148445336, + "loss": 1.4185, + "theoretical_loss": 3.8648190118886725, + "tokens_seen": 564133888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186960882647944, + "loss": 1.4829, + "theoretical_loss": 3.8647243099464106, + "tokens_seen": 564264960 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041867602808425274, + "loss": 1.6106, + "theoretical_loss": 3.8646296361575687, + "tokens_seen": 564396032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041865596790371115, + "loss": 1.3867, + "theoretical_loss": 3.8645349905072406, + "tokens_seen": 564527104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186359077231695, + "loss": 1.4051, + "theoretical_loss": 3.864440372980533, + "tokens_seen": 564658176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186158475426279, + "loss": 1.5175, + "theoretical_loss": 3.8643457835625634, + "tokens_seen": 564789248 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041859578736208624, + "loss": 1.4996, + "theoretical_loss": 3.864251222238462, + "tokens_seen": 564920320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185757271815446, + "loss": 1.4721, + "theoretical_loss": 3.864156688993369, + "tokens_seen": 565051392 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418555667001003, + "loss": 1.4107, + "theoretical_loss": 3.864062183812436, + "tokens_seen": 565182464 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.37713831663131714, + "objective/train/avg_token_score": 0.01374147180467844, + "objective/train/docs_used": 329612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2935733795166016, + "objective/train/original_loss": 3.755645275115967, + "objective/train/theoretical_loss": 3.864014941741392, + "objective/train/tokens_used": 565248000, + "objective/train/unlikelihood_loss": 0.1588522344827652, + "theoretical_loss": 3.864014941741392, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185356068204614, + "loss": 1.5042, + "theoretical_loss": 3.8639677066808256, + "tokens_seen": 565313536 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041851554663991974, + "loss": 1.5188, + "theoretical_loss": 3.8638732575837125, + "tokens_seen": 565444608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184954864593781, + "loss": 1.5116, + "theoretical_loss": 3.863778836506283, + "tokens_seen": 565575680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184754262788365, + "loss": 1.5429, + "theoretical_loss": 3.8636844434337325, + "tokens_seen": 565706752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184553660982949, + "loss": 1.6697, + "theoretical_loss": 3.863590078351271, + "tokens_seen": 565837824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041843530591775325, + "loss": 1.4738, + "theoretical_loss": 3.8634957412441167, + "tokens_seen": 565968896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184152457372116, + "loss": 1.5514, + "theoretical_loss": 3.8634014320975005, + "tokens_seen": 566099968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041839518555666997, + "loss": 1.5185, + "theoretical_loss": 3.863307150896664, + "tokens_seen": 566231040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183751253761284, + "loss": 1.4115, + "theoretical_loss": 3.8632128976268603, + "tokens_seen": 566362112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041835506519558675, + "loss": 1.4309, + "theoretical_loss": 3.8631186722733535, + "tokens_seen": 566493184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183350050150451, + "loss": 1.5064, + "theoretical_loss": 3.8630244748214198, + "tokens_seen": 566624256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183149448345035, + "loss": 1.5129, + "theoretical_loss": 3.862930305256345, + "tokens_seen": 566755328 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.4685361683368683, + "objective/train/avg_token_score": 0.020750796422362328, + "objective/train/docs_used": 331071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7844135761260986, + "objective/train/original_loss": 4.0298967361450195, + "objective/train/theoretical_loss": 3.8628361635634265, + "objective/train/tokens_used": 566886400, + "objective/train/unlikelihood_loss": 0.1526690572500229, + "theoretical_loss": 3.8628361635634265, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182948846539619, + "loss": 1.4882, + "theoretical_loss": 3.8628361635634265, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041827482447342025, + "loss": 1.5128, + "theoretical_loss": 3.8627420497279745, + "tokens_seen": 567017472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182547642928786, + "loss": 1.6453, + "theoretical_loss": 3.8626479637353075, + "tokens_seen": 567148544 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418234704112337, + "loss": 1.5273, + "theoretical_loss": 3.8625539055707576, + "tokens_seen": 567279616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182146439317954, + "loss": 1.5139, + "theoretical_loss": 3.862459875219667, + "tokens_seen": 567410688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041819458375125376, + "loss": 1.4496, + "theoretical_loss": 3.8623658726673886, + "tokens_seen": 567541760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181745235707121, + "loss": 1.5045, + "theoretical_loss": 3.862271897899287, + "tokens_seen": 567672832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041815446339017054, + "loss": 1.5649, + "theoretical_loss": 3.8621779509007377, + "tokens_seen": 567803904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181344032096289, + "loss": 1.4613, + "theoretical_loss": 3.862084031657128, + "tokens_seen": 567934976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181143430290873, + "loss": 1.5159, + "theoretical_loss": 3.8619901401538543, + "tokens_seen": 568066048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180942828485457, + "loss": 1.5407, + "theoretical_loss": 3.861896276376326, + "tokens_seen": 568197120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041807422266800404, + "loss": 1.4418, + "theoretical_loss": 3.8618024403099636, + "tokens_seen": 568328192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180541624874624, + "loss": 1.4974, + "theoretical_loss": 3.8617086319401968, + "tokens_seen": 568459264 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.3284457325935364, + "objective/train/avg_token_score": 0.011517138220369816, + "objective/train/docs_used": 331416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2020608186721802, + "objective/train/original_loss": 4.407777309417725, + "objective/train/theoretical_loss": 3.861661738136987, + "objective/train/tokens_used": 568524800, + "objective/train/unlikelihood_loss": 0.14563091099262238, + "theoretical_loss": 3.861661738136987, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180341023069208, + "loss": 1.498, + "theoretical_loss": 3.8616148512524675, + "tokens_seen": 568590336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180140421263792, + "loss": 1.51, + "theoretical_loss": 3.8615210982322288, + "tokens_seen": 568721408 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041799398194583754, + "loss": 1.59, + "theoretical_loss": 3.8614273728649446, + "tokens_seen": 568852480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179739217652959, + "loss": 1.426, + "theoretical_loss": 3.8613336751360894, + "tokens_seen": 568983552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041795386158475427, + "loss": 1.6531, + "theoretical_loss": 3.8612400050311493, + "tokens_seen": 569114624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179338014042127, + "loss": 1.4923, + "theoretical_loss": 3.861146362535621, + "tokens_seen": 569245696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041791374122367105, + "loss": 1.5301, + "theoretical_loss": 3.8610527476350116, + "tokens_seen": 569376768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178936810431294, + "loss": 1.4552, + "theoretical_loss": 3.86095916031484, + "tokens_seen": 569507840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041787362086258777, + "loss": 1.4689, + "theoretical_loss": 3.8608656005606354, + "tokens_seen": 569638912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178535606820462, + "loss": 1.5132, + "theoretical_loss": 3.8607720683579396, + "tokens_seen": 569769984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783350050150455, + "loss": 1.4345, + "theoretical_loss": 3.8606785636923027, + "tokens_seen": 569901056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178134403209629, + "loss": 1.5175, + "theoretical_loss": 3.8605850865492872, + "tokens_seen": 570032128 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.43047410249710083, + "objective/train/avg_token_score": 0.02064175345003605, + "objective/train/docs_used": 332819, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4973698854446411, + "objective/train/original_loss": 3.5845766067504883, + "objective/train/theoretical_loss": 3.8604916369144666, + "objective/train/tokens_used": 570163200, + "objective/train/unlikelihood_loss": 0.2515372037887573, + "theoretical_loss": 3.8604916369144666, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177933801404213, + "loss": 1.4534, + "theoretical_loss": 3.8604916369144666, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041777331995987964, + "loss": 1.6288, + "theoretical_loss": 3.860398214773425, + "tokens_seen": 570294272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041775325977933805, + "loss": 1.4872, + "theoretical_loss": 3.8603048201117565, + "tokens_seen": 570425344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177331995987964, + "loss": 1.4164, + "theoretical_loss": 3.8602114529150673, + "tokens_seen": 570556416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177131394182548, + "loss": 1.525, + "theoretical_loss": 3.8601181131689737, + "tokens_seen": 570687488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041769307923771314, + "loss": 1.438, + "theoretical_loss": 3.8600248008591045, + "tokens_seen": 570818560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041767301905717156, + "loss": 1.4738, + "theoretical_loss": 3.8599315159710965, + "tokens_seen": 570949632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176529588766299, + "loss": 1.6646, + "theoretical_loss": 3.8598382584905995, + "tokens_seen": 571080704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176328986960883, + "loss": 1.5638, + "theoretical_loss": 3.859745028403273, + "tokens_seen": 571211776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041761283851554664, + "loss": 1.6145, + "theoretical_loss": 3.859651825694787, + "tokens_seen": 571342848 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417592778335005, + "loss": 1.4472, + "theoretical_loss": 3.8595586503508246, + "tokens_seen": 571473920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175727181544634, + "loss": 1.3499, + "theoretical_loss": 3.8594655023570765, + "tokens_seen": 571604992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175526579739218, + "loss": 1.6236, + "theoretical_loss": 3.859372381699247, + "tokens_seen": 571736064 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.31506597995758057, + "objective/train/avg_token_score": 0.016959313303232193, + "objective/train/docs_used": 333556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9501151442527771, + "objective/train/original_loss": 3.510495901107788, + "objective/train/theoretical_loss": 3.859325831616836, + "objective/train/tokens_used": 571801600, + "objective/train/unlikelihood_loss": 0.19363252818584442, + "theoretical_loss": 3.859325831616836, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041753259779338015, + "loss": 1.4159, + "theoretical_loss": 3.859279288363048, + "tokens_seen": 571867136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175125376128385, + "loss": 1.6299, + "theoretical_loss": 3.8591862223342064, + "tokens_seen": 571998208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174924774322969, + "loss": 1.5366, + "theoretical_loss": 3.859093183598455, + "tokens_seen": 572129280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174724172517553, + "loss": 1.4375, + "theoretical_loss": 3.859000172141541, + "tokens_seen": 572260352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041745235707121365, + "loss": 1.5723, + "theoretical_loss": 3.858907187949221, + "tokens_seen": 572391424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417432296890672, + "loss": 1.6735, + "theoretical_loss": 3.858814231007262, + "tokens_seen": 572522496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041741223671013037, + "loss": 1.3602, + "theoretical_loss": 3.858721301301442, + "tokens_seen": 572653568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173921765295888, + "loss": 1.4342, + "theoretical_loss": 3.8586283988175496, + "tokens_seen": 572784640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041737211634904715, + "loss": 1.4224, + "theoretical_loss": 3.8585355235413843, + "tokens_seen": 572915712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173520561685055, + "loss": 1.5072, + "theoretical_loss": 3.858442675458756, + "tokens_seen": 573046784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173319959879639, + "loss": 1.5743, + "theoretical_loss": 3.858349854555486, + "tokens_seen": 573177856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173119358074223, + "loss": 1.5621, + "theoretical_loss": 3.8582570608174045, + "tokens_seen": 573308928 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.29276636242866516, + "objective/train/avg_token_score": 0.03536292165517807, + "objective/train/docs_used": 334721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1436461210250854, + "objective/train/original_loss": 4.6639227867126465, + "objective/train/theoretical_loss": 3.858164294230354, + "objective/train/tokens_used": 573440000, + "objective/train/unlikelihood_loss": 0.10083737224340439, + "theoretical_loss": 3.858164294230354, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041729187562688065, + "loss": 1.4795, + "theoretical_loss": 3.858164294230354, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417271815446339, + "loss": 1.5866, + "theoretical_loss": 3.8580715547801865, + "tokens_seen": 573571072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172517552657974, + "loss": 1.678, + "theoretical_loss": 3.8579788424527655, + "tokens_seen": 573702144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172316950852558, + "loss": 1.5994, + "theoretical_loss": 3.857886157233965, + "tokens_seen": 573833216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041721163490471416, + "loss": 1.4772, + "theoretical_loss": 3.857793499109669, + "tokens_seen": 573964288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171915747241725, + "loss": 1.5592, + "theoretical_loss": 3.8577008680657716, + "tokens_seen": 574095360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171715145436309, + "loss": 1.5461, + "theoretical_loss": 3.8576082640881797, + "tokens_seen": 574226432 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041715145436308924, + "loss": 1.5339, + "theoretical_loss": 3.8575156871628082, + "tokens_seen": 574357504 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041713139418254766, + "loss": 1.5063, + "theoretical_loss": 3.8574231372755836, + "tokens_seen": 574488576 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417111334002006, + "loss": 1.5664, + "theoretical_loss": 3.8573306144124437, + "tokens_seen": 574619648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170912738214644, + "loss": 1.4587, + "theoretical_loss": 3.8572381185593354, + "tokens_seen": 574750720 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041707121364092275, + "loss": 1.5352, + "theoretical_loss": 3.8571456497022165, + "tokens_seen": 574881792 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041705115346038116, + "loss": 1.7719, + "theoretical_loss": 3.8570532078270565, + "tokens_seen": 575012864 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.410740464925766, + "objective/train/avg_token_score": 0.012464967556297779, + "objective/train/docs_used": 335233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4862878322601318, + "objective/train/original_loss": 3.868055820465088, + "objective/train/theoretical_loss": 3.857006997003329, + "objective/train/tokens_used": 575078400, + "objective/train/unlikelihood_loss": 0.2018526941537857, + "theoretical_loss": 3.857006997003329, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170310932798395, + "loss": 1.5556, + "theoretical_loss": 3.8569607929198337, + "tokens_seen": 575143936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170110330992979, + "loss": 1.5718, + "theoretical_loss": 3.856868404966538, + "tokens_seen": 575275008 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041699097291875625, + "loss": 1.6003, + "theoretical_loss": 3.856776043953169, + "tokens_seen": 575406080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169709127382146, + "loss": 1.6744, + "theoretical_loss": 3.856683709865737, + "tokens_seen": 575537152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041695085255767303, + "loss": 1.4031, + "theoretical_loss": 3.856591402690264, + "tokens_seen": 575668224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169307923771314, + "loss": 1.5336, + "theoretical_loss": 3.856499122412781, + "tokens_seen": 575799296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041691073219658975, + "loss": 1.6005, + "theoretical_loss": 3.856406869019329, + "tokens_seen": 575930368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168906720160481, + "loss": 1.2878, + "theoretical_loss": 3.856314642495961, + "tokens_seen": 576061440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041687061183550653, + "loss": 1.4465, + "theoretical_loss": 3.8562224428287393, + "tokens_seen": 576192512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168505516549649, + "loss": 1.3374, + "theoretical_loss": 3.856130270003736, + "tokens_seen": 576323584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041683049147442326, + "loss": 1.6655, + "theoretical_loss": 3.8560381240070365, + "tokens_seen": 576454656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168104312938816, + "loss": 1.6154, + "theoretical_loss": 3.855946004824733, + "tokens_seen": 576585728 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.3080400824546814, + "objective/train/avg_token_score": 0.017052171751856804, + "objective/train/docs_used": 335838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9714576005935669, + "objective/train/original_loss": 3.790057897567749, + "objective/train/theoretical_loss": 3.85585391244293, + "objective/train/tokens_used": 576716800, + "objective/train/unlikelihood_loss": 0.24715007841587067, + "theoretical_loss": 3.85585391244293, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041679037111334, + "loss": 1.4714, + "theoretical_loss": 3.85585391244293, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167703109327984, + "loss": 1.5795, + "theoretical_loss": 3.8557618468477415, + "tokens_seen": 576847872 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041675025075225676, + "loss": 1.5285, + "theoretical_loss": 3.8556698080252936, + "tokens_seen": 576978944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167301905717151, + "loss": 1.4149, + "theoretical_loss": 3.8555777959617203, + "tokens_seen": 577110016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167101303911735, + "loss": 1.3468, + "theoretical_loss": 3.8554858106431675, + "tokens_seen": 577241088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166900702106319, + "loss": 1.5689, + "theoretical_loss": 3.8553938520557915, + "tokens_seen": 577372160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041667001003009026, + "loss": 1.5243, + "theoretical_loss": 3.8553019201857577, + "tokens_seen": 577503232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166499498495487, + "loss": 1.4107, + "theoretical_loss": 3.8552100150192423, + "tokens_seen": 577634304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041662988966900704, + "loss": 1.4474, + "theoretical_loss": 3.855118136542433, + "tokens_seen": 577765376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166098294884654, + "loss": 1.4431, + "theoretical_loss": 3.855026284741526, + "tokens_seen": 577896448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165897693079238, + "loss": 1.4762, + "theoretical_loss": 3.8549344596027293, + "tokens_seen": 578027520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165697091273822, + "loss": 1.3957, + "theoretical_loss": 3.8548426611122597, + "tokens_seen": 578158592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654964894684055, + "loss": 1.3942, + "theoretical_loss": 3.854750889256345, + "tokens_seen": 578289664 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.43218475580215454, + "objective/train/avg_token_score": 0.007149472367018461, + "objective/train/docs_used": 337192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.293778657913208, + "objective/train/original_loss": 4.05076265335083, + "objective/train/theoretical_loss": 3.854705013312045, + "objective/train/tokens_used": 578355200, + "objective/train/unlikelihood_loss": 0.13084176182746887, + "theoretical_loss": 3.854705013312045, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165295887662989, + "loss": 1.5097, + "theoretical_loss": 3.8546591440212237, + "tokens_seen": 578420736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165095285857573, + "loss": 1.4131, + "theoretical_loss": 3.8545674253931432, + "tokens_seen": 578551808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164894684052157, + "loss": 1.4509, + "theoretical_loss": 3.8544757333583632, + "tokens_seen": 578682880 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041646940822467405, + "loss": 1.3605, + "theoretical_loss": 3.854384067903151, + "tokens_seen": 578813952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164493480441324, + "loss": 1.5583, + "theoretical_loss": 3.8542924290137863, + "tokens_seen": 578945024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164292878635908, + "loss": 1.3599, + "theoretical_loss": 3.8542008166765576, + "tokens_seen": 579076096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164092276830492, + "loss": 1.4714, + "theoretical_loss": 3.8541092308777647, + "tokens_seen": 579207168 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638916750250755, + "loss": 1.5355, + "theoretical_loss": 3.8540176716037164, + "tokens_seen": 579338240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163691073219659, + "loss": 1.2708, + "theoretical_loss": 3.8539261388407327, + "tokens_seen": 579469312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163490471414243, + "loss": 1.5444, + "theoretical_loss": 3.853834632575143, + "tokens_seen": 579600384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163289869608827, + "loss": 1.4964, + "theoretical_loss": 3.853743152793287, + "tokens_seen": 579731456 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041630892678034105, + "loss": 1.4589, + "theoretical_loss": 3.8536516994815146, + "tokens_seen": 579862528 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.32331377267837524, + "objective/train/avg_token_score": 0.055630918592214584, + "objective/train/docs_used": 337878, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.113051414489746, + "objective/train/original_loss": 4.3019514083862305, + "objective/train/theoretical_loss": 3.8535602726261864, + "objective/train/tokens_used": 579993600, + "objective/train/unlikelihood_loss": 0.16012266278266907, + "theoretical_loss": 3.8535602726261864, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162888665997994, + "loss": 1.2567, + "theoretical_loss": 3.8535602726261864, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162688064192578, + "loss": 1.4235, + "theoretical_loss": 3.8534688722136723, + "tokens_seen": 580124672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162487462387162, + "loss": 1.544, + "theoretical_loss": 3.8533774982303526, + "tokens_seen": 580255744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041622868605817456, + "loss": 1.4034, + "theoretical_loss": 3.8532861506626173, + "tokens_seen": 580386816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162086258776329, + "loss": 1.5044, + "theoretical_loss": 3.8531948294968674, + "tokens_seen": 580517888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161885656970913, + "loss": 1.5761, + "theoretical_loss": 3.853103534719513, + "tokens_seen": 580648960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041616850551654964, + "loss": 1.4135, + "theoretical_loss": 3.8530122663169752, + "tokens_seen": 580780032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041614844533600806, + "loss": 1.41, + "theoretical_loss": 3.852921024275684, + "tokens_seen": 580911104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161283851554664, + "loss": 1.3965, + "theoretical_loss": 3.8528298085820802, + "tokens_seen": 581042176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161083249749248, + "loss": 1.4879, + "theoretical_loss": 3.852738619222615, + "tokens_seen": 581173248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041608826479438315, + "loss": 1.4855, + "theoretical_loss": 3.8526474561837483, + "tokens_seen": 581304320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041606820461384156, + "loss": 1.4659, + "theoretical_loss": 3.852556319451952, + "tokens_seen": 581435392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160481444332999, + "loss": 1.5415, + "theoretical_loss": 3.852465209013705, + "tokens_seen": 581566464 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.415628045797348, + "objective/train/avg_token_score": 0.01404473278671503, + "objective/train/docs_used": 339259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.375450611114502, + "objective/train/original_loss": 3.8198952674865723, + "objective/train/theoretical_loss": 3.8524196636504415, + "objective/train/tokens_used": 581632000, + "objective/train/unlikelihood_loss": 0.21155038475990295, + "theoretical_loss": 3.8524196636504415, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160280842527583, + "loss": 1.3202, + "theoretical_loss": 3.8523741248555, + "tokens_seen": 581697536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041600802407221665, + "loss": 1.3599, + "theoretical_loss": 3.852283066963836, + "tokens_seen": 581828608 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415987963891675, + "loss": 1.5542, + "theoretical_loss": 3.852192035325226, + "tokens_seen": 581959680 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041596790371113343, + "loss": 1.5125, + "theoretical_loss": 3.852101029926188, + "tokens_seen": 582090752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159478435305918, + "loss": 1.4721, + "theoretical_loss": 3.852010050753254, + "tokens_seen": 582221824 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041592778335005015, + "loss": 1.3842, + "theoretical_loss": 3.8519190977929645, + "tokens_seen": 582352896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159077231695085, + "loss": 1.4271, + "theoretical_loss": 3.85182817103187, + "tokens_seen": 582483968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041588766298896693, + "loss": 1.385, + "theoretical_loss": 3.85173727045653, + "tokens_seen": 582615040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158676028084253, + "loss": 1.3607, + "theoretical_loss": 3.851646396053516, + "tokens_seen": 582746112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041584754262788366, + "loss": 1.6908, + "theoretical_loss": 3.851555547809408, + "tokens_seen": 582877184 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415827482447342, + "loss": 1.5575, + "theoretical_loss": 3.8514647257107955, + "tokens_seen": 583008256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158074222668004, + "loss": 1.5368, + "theoretical_loss": 3.8513739297442786, + "tokens_seen": 583139328 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.37939882278442383, + "objective/train/avg_token_score": 0.012932621873915195, + "objective/train/docs_used": 340083, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3577672243118286, + "objective/train/original_loss": 3.419050931930542, + "objective/train/theoretical_loss": 3.851283159896468, + "objective/train/tokens_used": 583270400, + "objective/train/unlikelihood_loss": 0.294486939907074, + "theoretical_loss": 3.851283159896468, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157873620862588, + "loss": 1.4448, + "theoretical_loss": 3.851283159896468, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041576730190571716, + "loss": 1.4166, + "theoretical_loss": 3.8511924161539826, + "tokens_seen": 583401472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157472417251755, + "loss": 1.4534, + "theoretical_loss": 3.8511016985034523, + "tokens_seen": 583532544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157271815446339, + "loss": 1.5211, + "theoretical_loss": 3.8510110069315173, + "tokens_seen": 583663616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157071213640923, + "loss": 1.2879, + "theoretical_loss": 3.8509203414248256, + "tokens_seen": 583794688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041568706118355066, + "loss": 1.4893, + "theoretical_loss": 3.850829701970037, + "tokens_seen": 583925760 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415667001003009, + "loss": 1.2667, + "theoretical_loss": 3.85073908855382, + "tokens_seen": 584056832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156469408224674, + "loss": 1.4526, + "theoretical_loss": 3.8506485011628544, + "tokens_seen": 584187904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041562688064192575, + "loss": 1.4288, + "theoretical_loss": 3.850557939783828, + "tokens_seen": 584318976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041560682046138417, + "loss": 1.6138, + "theoretical_loss": 3.8504674044034393, + "tokens_seen": 584450048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041558676028084253, + "loss": 1.633, + "theoretical_loss": 3.850376895008396, + "tokens_seen": 584581120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155667001003009, + "loss": 1.4531, + "theoretical_loss": 3.850286411585417, + "tokens_seen": 584712192 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041554663991975925, + "loss": 1.4509, + "theoretical_loss": 3.8501959541212294, + "tokens_seen": 584843264 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.31414955854415894, + "objective/train/avg_token_score": 0.012615097686648369, + "objective/train/docs_used": 341398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1601176261901855, + "objective/train/original_loss": 4.111655235290527, + "objective/train/theoretical_loss": 3.850150735119537, + "objective/train/tokens_used": 584908800, + "objective/train/unlikelihood_loss": 0.15587183833122253, + "theoretical_loss": 3.850150735119537, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041552657973921767, + "loss": 1.4771, + "theoretical_loss": 3.8501055226025707, + "tokens_seen": 584974336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041550651955867603, + "loss": 1.454, + "theoretical_loss": 3.850015117016188, + "tokens_seen": 585105408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154864593781344, + "loss": 1.5285, + "theoretical_loss": 3.8499247373488377, + "tokens_seen": 585236480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041546639919759276, + "loss": 1.5182, + "theoretical_loss": 3.8498343835872872, + "tokens_seen": 585367552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154463390170511, + "loss": 1.5975, + "theoretical_loss": 3.8497440557183134, + "tokens_seen": 585498624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041542627883650954, + "loss": 1.6371, + "theoretical_loss": 3.849653753728701, + "tokens_seen": 585629696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154062186559679, + "loss": 1.5461, + "theoretical_loss": 3.8495634776052463, + "tokens_seen": 585760768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041538615847542626, + "loss": 1.2369, + "theoretical_loss": 3.849473227334755, + "tokens_seen": 585891840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153660982948846, + "loss": 1.4595, + "theoretical_loss": 3.849383002904043, + "tokens_seen": 586022912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041534603811434304, + "loss": 1.3872, + "theoretical_loss": 3.8492928042999335, + "tokens_seen": 586153984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153259779338014, + "loss": 1.4741, + "theoretical_loss": 3.849202631509262, + "tokens_seen": 586285056 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041530591775325976, + "loss": 1.3709, + "theoretical_loss": 3.8491124845188716, + "tokens_seen": 586416128 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.35477760434150696, + "objective/train/avg_token_score": 0.018234847113490105, + "objective/train/docs_used": 342071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2699406147003174, + "objective/train/original_loss": 4.259076118469238, + "objective/train/theoretical_loss": 3.8490223633156173, + "objective/train/tokens_used": 586547200, + "objective/train/unlikelihood_loss": 0.1685103178024292, + "theoretical_loss": 3.8490223633156173, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152858575727181, + "loss": 1.4075, + "theoretical_loss": 3.8490223633156173, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152657973921765, + "loss": 1.3445, + "theoretical_loss": 3.8489322678863624, + "tokens_seen": 586678272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152457372116349, + "loss": 1.4237, + "theoretical_loss": 3.848842198217979, + "tokens_seen": 586809344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041522567703109327, + "loss": 1.4361, + "theoretical_loss": 3.84875215429735, + "tokens_seen": 586940416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041520561685055163, + "loss": 1.3083, + "theoretical_loss": 3.8486621361113684, + "tokens_seen": 587071488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041518555667001, + "loss": 1.3499, + "theoretical_loss": 3.848572143646936, + "tokens_seen": 587202560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151654964894684, + "loss": 1.521, + "theoretical_loss": 3.8484821768909634, + "tokens_seen": 587333632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151454363089268, + "loss": 1.5277, + "theoretical_loss": 3.8483922358303717, + "tokens_seen": 587464704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151253761283852, + "loss": 1.4885, + "theoretical_loss": 3.8483023204520928, + "tokens_seen": 587595776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041510531594784355, + "loss": 1.5283, + "theoretical_loss": 3.848212430743065, + "tokens_seen": 587726848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041508525576730196, + "loss": 1.446, + "theoretical_loss": 3.848122566690239, + "tokens_seen": 587857920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150651955867603, + "loss": 1.4859, + "theoretical_loss": 3.8480327282805744, + "tokens_seen": 587988992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150451354062187, + "loss": 1.3935, + "theoretical_loss": 3.847942915501039, + "tokens_seen": 588120064 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4999389052391052, + "objective/train/avg_token_score": 0.008891026489436626, + "objective/train/docs_used": 342650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.745088815689087, + "objective/train/original_loss": 3.872380256652832, + "objective/train/theoretical_loss": 3.8478980187185003, + "objective/train/tokens_used": 588185600, + "objective/train/unlikelihood_loss": 0.1340382695198059, + "theoretical_loss": 3.8478980187185003, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041502507522567705, + "loss": 1.3831, + "theoretical_loss": 3.8478531283386124, + "tokens_seen": 588251136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150050150451354, + "loss": 1.4038, + "theoretical_loss": 3.847763366780281, + "tokens_seen": 588382208 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498495486459383, + "loss": 1.5453, + "theoretical_loss": 3.8476736308130426, + "tokens_seen": 588513280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149648946840522, + "loss": 1.4038, + "theoretical_loss": 3.8475839204239044, + "tokens_seen": 588644352 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041494483450351055, + "loss": 1.3637, + "theoretical_loss": 3.8474942355998825, + "tokens_seen": 588775424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149247743229689, + "loss": 1.4316, + "theoretical_loss": 3.8474045763280023, + "tokens_seen": 588906496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041490471414242733, + "loss": 1.6323, + "theoretical_loss": 3.8473149425953, + "tokens_seen": 589037568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148846539618857, + "loss": 1.4029, + "theoretical_loss": 3.8472253343888196, + "tokens_seen": 589168640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041486459378134406, + "loss": 1.4195, + "theoretical_loss": 3.8471357516956153, + "tokens_seen": 589299712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148445336008024, + "loss": 1.5306, + "theoretical_loss": 3.8470461945027505, + "tokens_seen": 589430784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148244734202608, + "loss": 1.4443, + "theoretical_loss": 3.8469566627972993, + "tokens_seen": 589561856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148044132397192, + "loss": 1.4823, + "theoretical_loss": 3.8468671565663426, + "tokens_seen": 589692928 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.5726417303085327, + "objective/train/avg_token_score": 0.010895898565649986, + "objective/train/docs_used": 343935, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0607388019561768, + "objective/train/original_loss": 3.6878914833068848, + "objective/train/theoretical_loss": 3.846777675796974, + "objective/train/tokens_used": 589824000, + "objective/train/unlikelihood_loss": 0.14677737653255463, + "theoretical_loss": 3.846777675796974, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041478435305917756, + "loss": 1.3588, + "theoretical_loss": 3.846777675796974, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147642928786359, + "loss": 1.5026, + "theoretical_loss": 3.8466882204762936, + "tokens_seen": 589955072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147442326980943, + "loss": 1.3238, + "theoretical_loss": 3.846598790591412, + "tokens_seen": 590086144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147241725175527, + "loss": 1.5089, + "theoretical_loss": 3.8465093861294504, + "tokens_seen": 590217216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041470411233701106, + "loss": 1.4212, + "theoretical_loss": 3.846420007077537, + "tokens_seen": 590348288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146840521564694, + "loss": 1.4739, + "theoretical_loss": 3.846330653422812, + "tokens_seen": 590479360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146639919759278, + "loss": 1.501, + "theoretical_loss": 3.8462413251524215, + "tokens_seen": 590610432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041464393179538615, + "loss": 1.5693, + "theoretical_loss": 3.8461520222535253, + "tokens_seen": 590741504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041462387161484457, + "loss": 1.5297, + "theoretical_loss": 3.846062744713289, + "tokens_seen": 590872576 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041460381143430293, + "loss": 1.4618, + "theoretical_loss": 3.8459734925188895, + "tokens_seen": 591003648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145837512537613, + "loss": 1.3261, + "theoretical_loss": 3.8458842656575123, + "tokens_seen": 591134720 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041456369107321965, + "loss": 1.5224, + "theoretical_loss": 3.8457950641163516, + "tokens_seen": 591265792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454363089267807, + "loss": 1.3278, + "theoretical_loss": 3.845705887882612, + "tokens_seen": 591396864 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4558895528316498, + "objective/train/avg_token_score": 0.015658829361200333, + "objective/train/docs_used": 344744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6233446598052979, + "objective/train/original_loss": 3.805819511413574, + "objective/train/theoretical_loss": 3.845661309252029, + "objective/train/tokens_used": 591462400, + "objective/train/unlikelihood_loss": 0.17303699254989624, + "theoretical_loss": 3.845661309252029, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041452357071213643, + "loss": 1.5114, + "theoretical_loss": 3.8456167369435073, + "tokens_seen": 591527936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145035105315948, + "loss": 1.6073, + "theoretical_loss": 3.84552761128626, + "tokens_seen": 591659008 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041448345035105316, + "loss": 1.5835, + "theoretical_loss": 3.845438510898102, + "tokens_seen": 591790080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144633901705115, + "loss": 1.2002, + "theoretical_loss": 3.8453494357662747, + "tokens_seen": 591921152 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041444332998996994, + "loss": 1.4201, + "theoretical_loss": 3.8452603858780288, + "tokens_seen": 592052224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144232698094283, + "loss": 1.5521, + "theoretical_loss": 3.8451713612206246, + "tokens_seen": 592183296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041440320962888666, + "loss": 1.5349, + "theoretical_loss": 3.84508236178133, + "tokens_seen": 592314368 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414383149448345, + "loss": 1.5031, + "theoretical_loss": 3.844993387547425, + "tokens_seen": 592445440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041436308926780344, + "loss": 1.5198, + "theoretical_loss": 3.8449044385061955, + "tokens_seen": 592576512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143430290872618, + "loss": 1.3538, + "theoretical_loss": 3.8448155146449396, + "tokens_seen": 592707584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041432296890672016, + "loss": 1.5557, + "theoretical_loss": 3.8447266159509628, + "tokens_seen": 592838656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143029087261785, + "loss": 1.4679, + "theoretical_loss": 3.84463774241158, + "tokens_seen": 592969728 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.3541666567325592, + "objective/train/avg_token_score": 0.016156869009137154, + "objective/train/docs_used": 345976, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.190863847732544, + "objective/train/original_loss": 3.7492716312408447, + "objective/train/theoretical_loss": 3.844548894014116, + "objective/train/tokens_used": 593100800, + "objective/train/unlikelihood_loss": 0.21690547466278076, + "theoretical_loss": 3.844548894014116, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142828485456369, + "loss": 1.5316, + "theoretical_loss": 3.844548894014116, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142627883650953, + "loss": 1.5358, + "theoretical_loss": 3.844460070745905, + "tokens_seen": 593231872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041424272818455367, + "loss": 1.4871, + "theoretical_loss": 3.8443712725942882, + "tokens_seen": 593362944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041422266800401203, + "loss": 1.5228, + "theoretical_loss": 3.844282499546619, + "tokens_seen": 593494016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142026078234704, + "loss": 1.7323, + "theoretical_loss": 3.8441937515902573, + "tokens_seen": 593625088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141825476429288, + "loss": 1.5594, + "theoretical_loss": 3.8441050287125744, + "tokens_seen": 593756160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041416248746238717, + "loss": 1.5317, + "theoretical_loss": 3.844016330900949, + "tokens_seen": 593887232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041414242728184553, + "loss": 1.5874, + "theoretical_loss": 3.843927658142771, + "tokens_seen": 594018304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141223671013039, + "loss": 1.3527, + "theoretical_loss": 3.843839010425436, + "tokens_seen": 594149376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141023069207623, + "loss": 1.5098, + "theoretical_loss": 3.8437503877363524, + "tokens_seen": 594280448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140822467402207, + "loss": 1.3849, + "theoretical_loss": 3.843661790062935, + "tokens_seen": 594411520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041406218655967904, + "loss": 1.3191, + "theoretical_loss": 3.84357321739261, + "tokens_seen": 594542592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140421263791374, + "loss": 1.4866, + "theoretical_loss": 3.8434846697128098, + "tokens_seen": 594673664 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.38501954078674316, + "objective/train/avg_token_score": 0.03767390176653862, + "objective/train/docs_used": 346603, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2809624671936035, + "objective/train/original_loss": 3.974229335784912, + "objective/train/theoretical_loss": 3.843440405240433, + "objective/train/tokens_used": 594739200, + "objective/train/unlikelihood_loss": 0.15178601443767548, + "theoretical_loss": 3.843440405240433, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041402206619859576, + "loss": 1.5946, + "theoretical_loss": 3.843396147010979, + "tokens_seen": 594804736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140020060180542, + "loss": 1.4606, + "theoretical_loss": 3.8433076492745695, + "tokens_seen": 594935808 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041398194583751254, + "loss": 1.6322, + "theoretical_loss": 3.8432191764910426, + "tokens_seen": 595066880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139618856569709, + "loss": 1.5872, + "theoretical_loss": 3.8431307286478678, + "tokens_seen": 595197952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041394182547642926, + "loss": 1.4489, + "theoretical_loss": 3.843042305732526, + "tokens_seen": 595329024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139217652958877, + "loss": 1.3802, + "theoretical_loss": 3.8429539077325043, + "tokens_seen": 595460096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041390170511534604, + "loss": 1.435, + "theoretical_loss": 3.842865534635301, + "tokens_seen": 595591168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138816449348044, + "loss": 1.4674, + "theoretical_loss": 3.842777186428423, + "tokens_seen": 595722240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041386158475426277, + "loss": 1.384, + "theoretical_loss": 3.8426888630993847, + "tokens_seen": 595853312 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041384152457372113, + "loss": 1.5661, + "theoretical_loss": 3.8426005646357106, + "tokens_seen": 595984384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041382146439317954, + "loss": 1.3068, + "theoretical_loss": 3.842512291024935, + "tokens_seen": 596115456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138014042126379, + "loss": 1.5246, + "theoretical_loss": 3.842424042254601, + "tokens_seen": 596246528 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.43139052391052246, + "objective/train/avg_token_score": 0.019480930641293526, + "objective/train/docs_used": 348035, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4563043117523193, + "objective/train/original_loss": 3.788301706314087, + "objective/train/theoretical_loss": 3.8423358183122582, + "objective/train/tokens_used": 596377600, + "objective/train/unlikelihood_loss": 0.16736803948879242, + "theoretical_loss": 3.8423358183122582, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041378134403209627, + "loss": 1.4645, + "theoretical_loss": 3.8423358183122582, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041376128385155463, + "loss": 1.3936, + "theoretical_loss": 3.8422476191854686, + "tokens_seen": 596508672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041374122367101305, + "loss": 1.3855, + "theoretical_loss": 3.8421594448618013, + "tokens_seen": 596639744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137211634904714, + "loss": 1.4532, + "theoretical_loss": 3.842071295328835, + "tokens_seen": 596770816 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041370110330992977, + "loss": 1.3864, + "theoretical_loss": 3.8419831705741556, + "tokens_seen": 596901888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041368104312938813, + "loss": 1.4822, + "theoretical_loss": 3.841895070585361, + "tokens_seen": 597032960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041366098294884655, + "loss": 1.6476, + "theoretical_loss": 3.8418069953500558, + "tokens_seen": 597164032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041364092276830497, + "loss": 1.649, + "theoretical_loss": 3.841718944855854, + "tokens_seen": 597295104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041362086258776333, + "loss": 1.372, + "theoretical_loss": 3.8416309190903783, + "tokens_seen": 597426176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136008024072217, + "loss": 1.3341, + "theoretical_loss": 3.8415429180412617, + "tokens_seen": 597557248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041358074222668005, + "loss": 1.3834, + "theoretical_loss": 3.8414549416961443, + "tokens_seen": 597688320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041356068204613847, + "loss": 1.5413, + "theoretical_loss": 3.841366990042676, + "tokens_seen": 597819392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041354062186559683, + "loss": 1.4008, + "theoretical_loss": 3.8412790630685154, + "tokens_seen": 597950464 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.28977271914482117, + "objective/train/avg_token_score": 0.01197260245680809, + "objective/train/docs_used": 348629, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9216458201408386, + "objective/train/original_loss": 3.9676473140716553, + "objective/train/theoretical_loss": 3.841235108832321, + "objective/train/tokens_used": 598016000, + "objective/train/unlikelihood_loss": 0.14898046851158142, + "theoretical_loss": 3.841235108832321, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135205616850552, + "loss": 1.4445, + "theoretical_loss": 3.84119116076133, + "tokens_seen": 598081536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041350050150451356, + "loss": 1.3845, + "theoretical_loss": 3.8411032831087963, + "tokens_seen": 598212608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134804413239719, + "loss": 1.4899, + "theoretical_loss": 3.841015430098599, + "tokens_seen": 598343680 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041346038114343034, + "loss": 1.4013, + "theoretical_loss": 3.8409276017184326, + "tokens_seen": 598474752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134403209628887, + "loss": 1.4651, + "theoretical_loss": 3.840839797956, + "tokens_seen": 598605824 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041342026078234706, + "loss": 1.292, + "theoretical_loss": 3.8407520187990127, + "tokens_seen": 598736896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134002006018054, + "loss": 1.5541, + "theoretical_loss": 3.8406642642351914, + "tokens_seen": 598867968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041338014042126384, + "loss": 1.4107, + "theoretical_loss": 3.840576534252265, + "tokens_seen": 598999040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133600802407222, + "loss": 1.5056, + "theoretical_loss": 3.840488828837973, + "tokens_seen": 599130112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041334002006018056, + "loss": 1.5114, + "theoretical_loss": 3.840401147980061, + "tokens_seen": 599261184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133199598796389, + "loss": 1.4337, + "theoretical_loss": 3.8403134916662847, + "tokens_seen": 599392256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132998996990973, + "loss": 1.3905, + "theoretical_loss": 3.8402258598844097, + "tokens_seen": 599523328 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.36742424964904785, + "objective/train/avg_token_score": 0.029935631901025772, + "objective/train/docs_used": 349996, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2140491008758545, + "objective/train/original_loss": 4.204592704772949, + "objective/train/theoretical_loss": 3.840138252622208, + "objective/train/tokens_used": 599654400, + "objective/train/unlikelihood_loss": 0.1415853351354599, + "theoretical_loss": 3.840138252622208, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132798395185557, + "loss": 1.3492, + "theoretical_loss": 3.840138252622208, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041325977933801407, + "loss": 1.5691, + "theoretical_loss": 3.8400506698674635, + "tokens_seen": 599785472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041323971915747243, + "loss": 1.4959, + "theoretical_loss": 3.839963111607965, + "tokens_seen": 599916544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132196589769308, + "loss": 1.4411, + "theoretical_loss": 3.8398755778315135, + "tokens_seen": 600047616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131995987963892, + "loss": 1.4184, + "theoretical_loss": 3.8397880685259165, + "tokens_seen": 600178688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041317953861584757, + "loss": 1.4997, + "theoretical_loss": 3.839700583678991, + "tokens_seen": 600309760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041315947843530593, + "loss": 1.4387, + "theoretical_loss": 3.8396131232785637, + "tokens_seen": 600440832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131394182547643, + "loss": 1.5275, + "theoretical_loss": 3.8395256873124683, + "tokens_seen": 600571904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131193580742227, + "loss": 1.4305, + "theoretical_loss": 3.8394382757685483, + "tokens_seen": 600702976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130992978936811, + "loss": 1.57, + "theoretical_loss": 3.8393508886346552, + "tokens_seen": 600834048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041307923771313944, + "loss": 1.3022, + "theoretical_loss": 3.83926352589865, + "tokens_seen": 600965120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130591775325978, + "loss": 1.4727, + "theoretical_loss": 3.839176187548402, + "tokens_seen": 601096192 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041303911735205616, + "loss": 1.4899, + "theoretical_loss": 3.8390888735717885, + "tokens_seen": 601227264 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4315738081932068, + "objective/train/avg_token_score": 0.012260204181075096, + "objective/train/docs_used": 350686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4201757907867432, + "objective/train/original_loss": 3.8342316150665283, + "objective/train/theoretical_loss": 3.8390452257198096, + "objective/train/tokens_used": 601292800, + "objective/train/unlikelihood_loss": 0.11259658634662628, + "theoretical_loss": 3.8390452257198096, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130190571715146, + "loss": 1.375, + "theoretical_loss": 3.839001583956697, + "tokens_seen": 601358336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041299899699097294, + "loss": 1.4189, + "theoretical_loss": 3.8389143186910224, + "tokens_seen": 601489408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129789368104313, + "loss": 1.5252, + "theoretical_loss": 3.838827077762668, + "tokens_seen": 601620480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041295887662988966, + "loss": 1.5391, + "theoretical_loss": 3.8387398611595476, + "tokens_seen": 601751552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129388164493481, + "loss": 1.6135, + "theoretical_loss": 3.838652668869581, + "tokens_seen": 601882624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041291875626880644, + "loss": 1.4085, + "theoretical_loss": 3.838565500880699, + "tokens_seen": 602013696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128986960882648, + "loss": 1.4676, + "theoretical_loss": 3.83847835718084, + "tokens_seen": 602144768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041287863590772317, + "loss": 1.3773, + "theoretical_loss": 3.8383912377579508, + "tokens_seen": 602275840 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041285857572718153, + "loss": 1.6076, + "theoretical_loss": 3.8383041425999864, + "tokens_seen": 602406912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041283851554663995, + "loss": 1.3761, + "theoretical_loss": 3.8382170716949124, + "tokens_seen": 602537984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128184553660983, + "loss": 1.3649, + "theoretical_loss": 3.8381300250307007, + "tokens_seen": 602669056 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041279839518555667, + "loss": 1.5752, + "theoretical_loss": 3.8380430025953327, + "tokens_seen": 602800128 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4177052676677704, + "objective/train/avg_token_score": 0.011318889446556568, + "objective/train/docs_used": 352047, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3438571691513062, + "objective/train/original_loss": 3.8496592044830322, + "objective/train/theoretical_loss": 3.837956004376799, + "objective/train/tokens_used": 602931200, + "objective/train/unlikelihood_loss": 0.19700142741203308, + "theoretical_loss": 3.837956004376799, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041277833500501503, + "loss": 1.6608, + "theoretical_loss": 3.837956004376799, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041275827482447345, + "loss": 1.441, + "theoretical_loss": 3.8378690303630973, + "tokens_seen": 603062272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127382146439318, + "loss": 1.4288, + "theoretical_loss": 3.837782080542236, + "tokens_seen": 603193344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041271815446339017, + "loss": 1.4916, + "theoretical_loss": 3.837695154902229, + "tokens_seen": 603324416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041269809428284854, + "loss": 1.3033, + "theoretical_loss": 3.8376082534311022, + "tokens_seen": 603455488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126780341023069, + "loss": 1.3031, + "theoretical_loss": 3.8375213761168867, + "tokens_seen": 603586560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126579739217653, + "loss": 1.5044, + "theoretical_loss": 3.8374345229476248, + "tokens_seen": 603717632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126379137412237, + "loss": 1.5035, + "theoretical_loss": 3.837347693911366, + "tokens_seen": 603848704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041261785356068204, + "loss": 1.4344, + "theoretical_loss": 3.837260888996169, + "tokens_seen": 603979776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125977933801404, + "loss": 1.5352, + "theoretical_loss": 3.8371741081900996, + "tokens_seen": 604110848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125777331995988, + "loss": 1.5946, + "theoretical_loss": 3.837087351481234, + "tokens_seen": 604241920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125576730190572, + "loss": 1.4719, + "theoretical_loss": 3.8370006188576555, + "tokens_seen": 604372992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041253761283851554, + "loss": 1.2678, + "theoretical_loss": 3.836913910307456, + "tokens_seen": 604504064 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.3923509418964386, + "objective/train/avg_token_score": 0.010112078860402107, + "objective/train/docs_used": 352465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4317771196365356, + "objective/train/original_loss": 4.348731994628906, + "objective/train/theoretical_loss": 3.836870565056154, + "objective/train/tokens_used": 604569600, + "objective/train/unlikelihood_loss": 0.11279493570327759, + "theoretical_loss": 3.836870565056154, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125175526579739, + "loss": 1.5122, + "theoretical_loss": 3.8368272258187366, + "tokens_seen": 604635136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041249749247743227, + "loss": 1.2894, + "theoretical_loss": 3.836740565379606, + "tokens_seen": 604766208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124774322968907, + "loss": 1.4996, + "theoretical_loss": 3.836653928978182, + "tokens_seen": 604897280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041245737211634904, + "loss": 1.3454, + "theoretical_loss": 3.8365673166025913, + "tokens_seen": 605028352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124373119358074, + "loss": 1.3027, + "theoretical_loss": 3.8364807282409674, + "tokens_seen": 605159424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041241725175526577, + "loss": 1.5057, + "theoretical_loss": 3.8363941638814536, + "tokens_seen": 605290496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123971915747242, + "loss": 1.3416, + "theoretical_loss": 3.8363076235122016, + "tokens_seen": 605421568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041237713139418255, + "loss": 1.4237, + "theoretical_loss": 3.83622110712137, + "tokens_seen": 605552640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123570712136409, + "loss": 1.3262, + "theoretical_loss": 3.8361346146971282, + "tokens_seen": 605683712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041233701103309927, + "loss": 1.4146, + "theoretical_loss": 3.836048146227652, + "tokens_seen": 605814784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041231695085255763, + "loss": 1.5451, + "theoretical_loss": 3.8359617017011267, + "tokens_seen": 605945856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041229689067201605, + "loss": 1.3069, + "theoretical_loss": 3.8358752811057455, + "tokens_seen": 606076928 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.47061339020729065, + "objective/train/avg_token_score": 0.003165400819852948, + "objective/train/docs_used": 353634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.681175708770752, + "objective/train/original_loss": 3.5376367568969727, + "objective/train/theoretical_loss": 3.8357888844297094, + "objective/train/tokens_used": 606208000, + "objective/train/unlikelihood_loss": 0.1784207969903946, + "theoretical_loss": 3.8357888844297094, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122768304914744, + "loss": 1.3184, + "theoretical_loss": 3.8357888844297094, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122567703109328, + "loss": 1.5651, + "theoretical_loss": 3.8357025116612293, + "tokens_seen": 606339072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041223671013039114, + "loss": 1.5169, + "theoretical_loss": 3.8356161627885235, + "tokens_seen": 606470144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041221664994984955, + "loss": 1.2234, + "theoretical_loss": 3.835529837799818, + "tokens_seen": 606601216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121965897693079, + "loss": 1.5053, + "theoretical_loss": 3.835443536683349, + "tokens_seen": 606732288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121765295887663, + "loss": 1.4598, + "theoretical_loss": 3.8353572594273597, + "tokens_seen": 606863360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121564694082247, + "loss": 1.5847, + "theoretical_loss": 3.835271006020101, + "tokens_seen": 606994432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121364092276831, + "loss": 1.3336, + "theoretical_loss": 3.8351847764498332, + "tokens_seen": 607125504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121163490471415, + "loss": 1.3501, + "theoretical_loss": 3.835098570704826, + "tokens_seen": 607256576 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041209628886659984, + "loss": 1.4099, + "theoretical_loss": 3.8350123887733543, + "tokens_seen": 607387648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120762286860582, + "loss": 1.3004, + "theoretical_loss": 3.8349262306437044, + "tokens_seen": 607518720 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041205616850551656, + "loss": 1.464, + "theoretical_loss": 3.8348400963041684, + "tokens_seen": 607649792 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412036108324975, + "loss": 1.4531, + "theoretical_loss": 3.834753985743049, + "tokens_seen": 607780864 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.2646627426147461, + "objective/train/avg_token_score": 0.01583893597126007, + "objective/train/docs_used": 354263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9999536871910095, + "objective/train/original_loss": 4.469597816467285, + "objective/train/theoretical_loss": 3.834710939375742, + "objective/train/tokens_used": 607846400, + "objective/train/unlikelihood_loss": 0.14511200785636902, + "theoretical_loss": 3.834710939375742, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041201604814443334, + "loss": 1.4492, + "theoretical_loss": 3.8346678989486556, + "tokens_seen": 607911936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119959879638917, + "loss": 1.3742, + "theoretical_loss": 3.8345818359093062, + "tokens_seen": 608043008 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041197592778335006, + "loss": 1.4287, + "theoretical_loss": 3.834495796613327, + "tokens_seen": 608174080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119558676028085, + "loss": 1.4213, + "theoretical_loss": 3.834409781049053, + "tokens_seen": 608305152 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041193580742226684, + "loss": 1.5101, + "theoretical_loss": 3.8343237892048263, + "tokens_seen": 608436224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119157472417252, + "loss": 1.4667, + "theoretical_loss": 3.8342378210689994, + "tokens_seen": 608567296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041189568706118357, + "loss": 1.4083, + "theoretical_loss": 3.83415187662993, + "tokens_seen": 608698368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041187562688064193, + "loss": 1.463, + "theoretical_loss": 3.834065955875987, + "tokens_seen": 608829440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041185556670010035, + "loss": 1.2794, + "theoretical_loss": 3.833980058795545, + "tokens_seen": 608960512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118355065195587, + "loss": 1.5537, + "theoretical_loss": 3.8338941853769892, + "tokens_seen": 609091584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041181544633901707, + "loss": 1.5483, + "theoretical_loss": 3.8338083356087105, + "tokens_seen": 609222656 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041179538615847543, + "loss": 1.5858, + "theoretical_loss": 3.8337225094791103, + "tokens_seen": 609353728 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4015762507915497, + "objective/train/avg_token_score": 0.020319471135735512, + "objective/train/docs_used": 355721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4089401960372925, + "objective/train/original_loss": 4.095301628112793, + "objective/train/theoretical_loss": 3.8336367069765958, + "objective/train/tokens_used": 609484800, + "objective/train/unlikelihood_loss": 0.2028389275074005, + "theoretical_loss": 3.8336367069765958, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041177532597793385, + "loss": 1.273, + "theoretical_loss": 3.8336367069765958, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117552657973922, + "loss": 1.45, + "theoretical_loss": 3.8335509280895854, + "tokens_seen": 609615872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117352056168506, + "loss": 1.4615, + "theoretical_loss": 3.833465172806503, + "tokens_seen": 609746944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041171514543630894, + "loss": 1.4819, + "theoretical_loss": 3.833379441115781, + "tokens_seen": 609878016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116950852557673, + "loss": 1.3545, + "theoretical_loss": 3.8332937330058616, + "tokens_seen": 610009088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116750250752257, + "loss": 1.4763, + "theoretical_loss": 3.8332080484651945, + "tokens_seen": 610140160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116549648946841, + "loss": 1.5326, + "theoretical_loss": 3.833122387482236, + "tokens_seen": 610271232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041163490471414244, + "loss": 1.3957, + "theoretical_loss": 3.833036750045452, + "tokens_seen": 610402304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004116148445336008, + "loss": 1.4854, + "theoretical_loss": 3.8329511361433166, + "tokens_seen": 610533376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115947843530592, + "loss": 1.4062, + "theoretical_loss": 3.832865545764311, + "tokens_seen": 610664448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115747241725176, + "loss": 1.4865, + "theoretical_loss": 3.832779978896926, + "tokens_seen": 610795520 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041155466399197594, + "loss": 1.4416, + "theoretical_loss": 3.8326944355296595, + "tokens_seen": 610926592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115346038114343, + "loss": 1.4191, + "theoretical_loss": 3.832608915651017, + "tokens_seen": 611057664 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4571114480495453, + "objective/train/avg_token_score": 0.008640832267701626, + "objective/train/docs_used": 356348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6731990575790405, + "objective/train/original_loss": 3.7301838397979736, + "objective/train/theoretical_loss": 3.83256616451634, + "objective/train/tokens_used": 611123200, + "objective/train/unlikelihood_loss": 0.17205381393432617, + "theoretical_loss": 3.83256616451634, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041151454363089267, + "loss": 1.4817, + "theoretical_loss": 3.8325234192495126, + "tokens_seen": 611188736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114944834503511, + "loss": 1.4969, + "theoretical_loss": 3.8324379463136697, + "tokens_seen": 611319808 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041147442326980944, + "loss": 1.4105, + "theoretical_loss": 3.832352496832018, + "tokens_seen": 611450880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114543630892678, + "loss": 1.5492, + "theoretical_loss": 3.8322670707930957, + "tokens_seen": 611581952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041143430290872617, + "loss": 1.5112, + "theoretical_loss": 3.83218166818545, + "tokens_seen": 611713024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114142427281846, + "loss": 1.2652, + "theoretical_loss": 3.8320962889976347, + "tokens_seen": 611844096 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041139418254764295, + "loss": 1.58, + "theoretical_loss": 3.8320109332182124, + "tokens_seen": 611975168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113741223671013, + "loss": 1.4196, + "theoretical_loss": 3.831925600835754, + "tokens_seen": 612106240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041135406218655967, + "loss": 1.4769, + "theoretical_loss": 3.8318402918388386, + "tokens_seen": 612237312 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041133400200601803, + "loss": 1.506, + "theoretical_loss": 3.831755006216052, + "tokens_seen": 612368384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041131394182547645, + "loss": 1.5132, + "theoretical_loss": 3.831669743955989, + "tokens_seen": 612499456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112938816449348, + "loss": 1.3302, + "theoretical_loss": 3.8315845050472532, + "tokens_seen": 612630528 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4703079164028168, + "objective/train/avg_token_score": 0.01573745161294937, + "objective/train/docs_used": 356825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7282345294952393, + "objective/train/original_loss": 3.63925838470459, + "objective/train/theoretical_loss": 3.8314992894784536, + "objective/train/tokens_used": 612761600, + "objective/train/unlikelihood_loss": 0.17425817251205444, + "theoretical_loss": 3.8314992894784536, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112738214643932, + "loss": 1.4771, + "theoretical_loss": 3.8314992894784536, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125376128385154, + "loss": 1.4603, + "theoretical_loss": 3.8314140972382105, + "tokens_seen": 612892672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041123370110330995, + "loss": 1.2944, + "theoretical_loss": 3.8313289283151497, + "tokens_seen": 613023744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112136409227683, + "loss": 1.3469, + "theoretical_loss": 3.831243782697906, + "tokens_seen": 613154816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111935807422267, + "loss": 1.2951, + "theoretical_loss": 3.831158660375122, + "tokens_seen": 613285888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041117352056168504, + "loss": 1.3934, + "theoretical_loss": 3.8310735613354483, + "tokens_seen": 613416960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111534603811434, + "loss": 1.2533, + "theoretical_loss": 3.830988485567543, + "tokens_seen": 613548032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111334002006018, + "loss": 1.4875, + "theoretical_loss": 3.830903433060073, + "tokens_seen": 613679104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111133400200602, + "loss": 1.4649, + "theoretical_loss": 3.830818403801713, + "tokens_seen": 613810176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041109327983951854, + "loss": 1.2906, + "theoretical_loss": 3.830733397781144, + "tokens_seen": 613941248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110732196589769, + "loss": 1.2456, + "theoretical_loss": 3.830648414987057, + "tokens_seen": 614072320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110531594784353, + "loss": 1.2733, + "theoretical_loss": 3.830563455408151, + "tokens_seen": 614203392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110330992978937, + "loss": 1.3083, + "theoretical_loss": 3.8304785190331305, + "tokens_seen": 614334464 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.37115103006362915, + "objective/train/avg_token_score": 0.012984885834157467, + "objective/train/docs_used": 357491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3638612031936646, + "objective/train/original_loss": 3.861525535583496, + "objective/train/theoretical_loss": 3.830436059543551, + "objective/train/tokens_used": 614400000, + "objective/train/unlikelihood_loss": 0.16821658611297607, + "theoretical_loss": 3.830436059543551, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041101303911735205, + "loss": 1.7196, + "theoretical_loss": 3.830393605850711, + "tokens_seen": 614465536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109929789368104, + "loss": 1.1773, + "theoretical_loss": 3.8303087158496134, + "tokens_seen": 614596608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041097291875626877, + "loss": 1.173, + "theoretical_loss": 3.8302238490185676, + "tokens_seen": 614727680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109528585757272, + "loss": 1.4127, + "theoretical_loss": 3.830139005346312, + "tokens_seen": 614858752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041093279839518555, + "loss": 1.2904, + "theoretical_loss": 3.8300541848215905, + "tokens_seen": 614989824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109127382146439, + "loss": 1.4453, + "theoretical_loss": 3.8299693874331573, + "tokens_seen": 615120896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108926780341023, + "loss": 1.3533, + "theoretical_loss": 3.8298846131697744, + "tokens_seen": 615251968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108726178535607, + "loss": 1.201, + "theoretical_loss": 3.8297998620202094, + "tokens_seen": 615383040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041085255767301905, + "loss": 1.4987, + "theoretical_loss": 3.8297151339732407, + "tokens_seen": 615514112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108324974924774, + "loss": 1.3022, + "theoretical_loss": 3.8296304290176524, + "tokens_seen": 615645184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108124373119358, + "loss": 1.5003, + "theoretical_loss": 3.8295457471422365, + "tokens_seen": 615776256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107923771313942, + "loss": 1.3209, + "theoretical_loss": 3.8294610883357945, + "tokens_seen": 615907328 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2765762507915497, + "objective/train/avg_token_score": 0.022694801911711693, + "objective/train/docs_used": 358741, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9784660339355469, + "objective/train/original_loss": 3.9417734146118164, + "objective/train/theoretical_loss": 3.829376452587134, + "objective/train/tokens_used": 616038400, + "objective/train/unlikelihood_loss": 0.2532103657722473, + "theoretical_loss": 3.829376452587134, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041077231695085256, + "loss": 1.4609, + "theoretical_loss": 3.829376452587134, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107522567703109, + "loss": 1.4066, + "theoretical_loss": 3.8292918398850713, + "tokens_seen": 616169472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107321965897693, + "loss": 1.3902, + "theoretical_loss": 3.82920725021843, + "tokens_seen": 616300544 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041071213640922764, + "loss": 1.2997, + "theoretical_loss": 3.829122683576042, + "tokens_seen": 616431616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041069207622868606, + "loss": 1.4858, + "theoretical_loss": 3.8290381399467464, + "tokens_seen": 616562688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106720160481444, + "loss": 1.3344, + "theoretical_loss": 3.8289536193193907, + "tokens_seen": 616693760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041065195586760284, + "loss": 1.4025, + "theoretical_loss": 3.8288691216828297, + "tokens_seen": 616824832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106318956870612, + "loss": 1.4296, + "theoretical_loss": 3.8287846470259264, + "tokens_seen": 616955904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106118355065196, + "loss": 1.552, + "theoretical_loss": 3.828700195337551, + "tokens_seen": 617086976 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410591775325978, + "loss": 1.3554, + "theoretical_loss": 3.8286157666065828, + "tokens_seen": 617218048 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041057171514543634, + "loss": 1.3402, + "theoretical_loss": 3.8285313608219056, + "tokens_seen": 617349120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105516549648947, + "loss": 1.3052, + "theoretical_loss": 3.8284469779724155, + "tokens_seen": 617480192 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041053159478435307, + "loss": 1.3052, + "theoretical_loss": 3.828362618047013, + "tokens_seen": 617611264 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.46590909361839294, + "objective/train/avg_token_score": 0.011999816633760929, + "objective/train/docs_used": 359404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5094083547592163, + "objective/train/original_loss": 3.980893135070801, + "objective/train/theoretical_loss": 3.828320446677378, + "objective/train/tokens_used": 617676800, + "objective/train/unlikelihood_loss": 0.12393984198570251, + "theoretical_loss": 3.828320446677378, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105115346038115, + "loss": 1.4529, + "theoretical_loss": 3.8282782810346063, + "tokens_seen": 617742336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041049147442326985, + "loss": 1.3784, + "theoretical_loss": 3.8281939669241143, + "tokens_seen": 617873408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104714142427282, + "loss": 1.3825, + "theoretical_loss": 3.82810967570446, + "tokens_seen": 618004480 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041045135406218657, + "loss": 1.3919, + "theoretical_loss": 3.828025407364577, + "tokens_seen": 618135552 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410431293881645, + "loss": 1.4041, + "theoretical_loss": 3.8279411618934045, + "tokens_seen": 618266624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041041123370110335, + "loss": 1.4088, + "theoretical_loss": 3.82785693927989, + "tokens_seen": 618397696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103911735205617, + "loss": 1.4916, + "theoretical_loss": 3.82777273951299, + "tokens_seen": 618528768 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041037111334002007, + "loss": 1.5682, + "theoretical_loss": 3.8276885625816663, + "tokens_seen": 618659840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041035105315947844, + "loss": 1.538, + "theoretical_loss": 3.8276044084748904, + "tokens_seen": 618790912 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041033099297893685, + "loss": 1.3839, + "theoretical_loss": 3.827520277181641, + "tokens_seen": 618921984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103109327983952, + "loss": 1.5539, + "theoretical_loss": 3.8274361686909035, + "tokens_seen": 619053056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102908726178536, + "loss": 1.4889, + "theoretical_loss": 3.827352082991672, + "tokens_seen": 619184128 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.3732893466949463, + "objective/train/avg_token_score": 0.019104547798633575, + "objective/train/docs_used": 360794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2005456686019897, + "objective/train/original_loss": 3.98445725440979, + "objective/train/theoretical_loss": 3.827268020072948, + "objective/train/tokens_used": 619315200, + "objective/train/unlikelihood_loss": 0.14163470268249512, + "theoretical_loss": 3.827268020072948, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041027081243731194, + "loss": 1.3425, + "theoretical_loss": 3.827268020072948, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041025075225677035, + "loss": 1.299, + "theoretical_loss": 3.8271839799237397, + "tokens_seen": 619446272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102306920762287, + "loss": 1.6187, + "theoretical_loss": 3.8270999625330644, + "tokens_seen": 619577344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102106318956871, + "loss": 1.3738, + "theoretical_loss": 3.8270159678899462, + "tokens_seen": 619708416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041019057171514544, + "loss": 1.3923, + "theoretical_loss": 3.8269319959834167, + "tokens_seen": 619839488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101705115346038, + "loss": 1.4009, + "theoretical_loss": 3.8268480468025157, + "tokens_seen": 619970560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101504513540622, + "loss": 1.5065, + "theoretical_loss": 3.82676412033629, + "tokens_seen": 620101632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101303911735206, + "loss": 1.3505, + "theoretical_loss": 3.826680216573795, + "tokens_seen": 620232704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041011033099297894, + "loss": 1.3851, + "theoretical_loss": 3.8265963355040915, + "tokens_seen": 620363776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100902708124373, + "loss": 1.7003, + "theoretical_loss": 3.8265124771162506, + "tokens_seen": 620494848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100702106318957, + "loss": 1.255, + "theoretical_loss": 3.8264286413993496, + "tokens_seen": 620625920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100501504513541, + "loss": 1.5705, + "theoretical_loss": 3.8263448283424726, + "tokens_seen": 620756992 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041003009027081245, + "loss": 1.3712, + "theoretical_loss": 3.826261037934713, + "tokens_seen": 620888064 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.39937683939933777, + "objective/train/avg_token_score": 0.00654844893142581, + "objective/train/docs_used": 361341, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4888705015182495, + "objective/train/original_loss": 4.043774604797363, + "objective/train/theoretical_loss": 3.8262191512208448, + "objective/train/tokens_used": 620953600, + "objective/train/unlikelihood_loss": 0.12301285564899445, + "theoretical_loss": 3.8262191512208448, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100100300902708, + "loss": 1.3344, + "theoretical_loss": 3.8261772701651697, + "tokens_seen": 621019136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040998996990972917, + "loss": 1.5005, + "theoretical_loss": 3.8260935250229515, + "tokens_seen": 621150208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099699097291876, + "loss": 1.3902, + "theoretical_loss": 3.8260098024971727, + "tokens_seen": 621281280 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040994984954864595, + "loss": 1.4164, + "theoretical_loss": 3.8259261025769566, + "tokens_seen": 621412352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099297893681043, + "loss": 1.4026, + "theoretical_loss": 3.825842425251434, + "tokens_seen": 621543424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099097291875627, + "loss": 1.4462, + "theoretical_loss": 3.8257587705097404, + "tokens_seen": 621674496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098896690070211, + "loss": 1.5376, + "theoretical_loss": 3.8256751383410235, + "tokens_seen": 621805568 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040986960882647945, + "loss": 1.3613, + "theoretical_loss": 3.825591528734434, + "tokens_seen": 621936640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098495486459378, + "loss": 1.3915, + "theoretical_loss": 3.8255079416791338, + "tokens_seen": 622067712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098294884653962, + "loss": 1.3391, + "theoretical_loss": 3.8254243771642895, + "tokens_seen": 622198784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098094282848546, + "loss": 1.5791, + "theoretical_loss": 3.8253408351790767, + "tokens_seen": 622329856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040978936810431296, + "loss": 1.4436, + "theoretical_loss": 3.825257315712678, + "tokens_seen": 622460928 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2656402587890625, + "objective/train/avg_token_score": 0.020830322057008743, + "objective/train/docs_used": 362042, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8504209518432617, + "objective/train/original_loss": 4.423110008239746, + "objective/train/theoretical_loss": 3.8251738187542843, + "objective/train/tokens_used": 622592000, + "objective/train/unlikelihood_loss": 0.12244540452957153, + "theoretical_loss": 3.8251738187542843, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097693079237713, + "loss": 1.261, + "theoretical_loss": 3.8251738187542843, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097492477432297, + "loss": 1.2915, + "theoretical_loss": 3.8250903442930917, + "tokens_seen": 622723072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040972918756268804, + "loss": 1.4155, + "theoretical_loss": 3.8250068923183065, + "tokens_seen": 622854144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040970912738214646, + "loss": 1.3704, + "theoretical_loss": 3.8249234628191404, + "tokens_seen": 622985216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096890672016048, + "loss": 1.4364, + "theoretical_loss": 3.8248400557848146, + "tokens_seen": 623116288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096690070210632, + "loss": 1.2654, + "theoretical_loss": 3.824756671204555, + "tokens_seen": 623247360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040964894684052155, + "loss": 1.1298, + "theoretical_loss": 3.824673309067598, + "tokens_seen": 623378432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040962888665997996, + "loss": 1.4316, + "theoretical_loss": 3.824589969363184, + "tokens_seen": 623509504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096088264794383, + "loss": 1.4605, + "theoretical_loss": 3.824506652080564, + "tokens_seen": 623640576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095887662988967, + "loss": 1.4785, + "theoretical_loss": 3.8244233572089943, + "tokens_seen": 623771648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040956870611835505, + "loss": 1.4477, + "theoretical_loss": 3.8243400847377407, + "tokens_seen": 623902720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095486459378134, + "loss": 1.2426, + "theoretical_loss": 3.8242568346560737, + "tokens_seen": 624033792 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040952858575727183, + "loss": 1.4633, + "theoretical_loss": 3.8241736069532726, + "tokens_seen": 624164864 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.38581377267837524, + "objective/train/avg_token_score": 0.025465743616223335, + "objective/train/docs_used": 363016, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5282154083251953, + "objective/train/original_loss": 4.084991931915283, + "objective/train/theoretical_loss": 3.8241320014905993, + "objective/train/tokens_used": 624230400, + "objective/train/unlikelihood_loss": 0.14625848829746246, + "theoretical_loss": 3.8241320014905993, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095085255767302, + "loss": 1.3288, + "theoretical_loss": 3.824090401618625, + "tokens_seen": 624295936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040948846539618855, + "loss": 1.6178, + "theoretical_loss": 3.8240072186414245, + "tokens_seen": 624427008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094684052156469, + "loss": 1.4476, + "theoretical_loss": 3.8239240580109723, + "tokens_seen": 624558080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040944834503510533, + "loss": 1.3995, + "theoretical_loss": 3.8238409197165772, + "tokens_seen": 624689152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094282848545637, + "loss": 1.362, + "theoretical_loss": 3.8237578037475553, + "tokens_seen": 624820224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040940822467402206, + "loss": 1.424, + "theoretical_loss": 3.823674710093231, + "tokens_seen": 624951296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093881644934804, + "loss": 1.4847, + "theoretical_loss": 3.8235916387429336, + "tokens_seen": 625082368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093681043129388, + "loss": 1.348, + "theoretical_loss": 3.8235085896860017, + "tokens_seen": 625213440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093480441323972, + "loss": 1.322, + "theoretical_loss": 3.8234255629117815, + "tokens_seen": 625344512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040932798395185556, + "loss": 1.3699, + "theoretical_loss": 3.8233425584096254, + "tokens_seen": 625475584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093079237713139, + "loss": 1.385, + "theoretical_loss": 3.8232595761688932, + "tokens_seen": 625606656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092878635907723, + "loss": 1.4204, + "theoretical_loss": 3.8231766161789524, + "tokens_seen": 625737728 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4359726309776306, + "objective/train/avg_token_score": 0.016611715778708458, + "objective/train/docs_used": 363672, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.319413423538208, + "objective/train/original_loss": 3.72617506980896, + "objective/train/theoretical_loss": 3.8230936784291787, + "objective/train/tokens_used": 625868800, + "objective/train/unlikelihood_loss": 0.15357278287410736, + "theoretical_loss": 3.8230936784291787, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092678034102307, + "loss": 1.3302, + "theoretical_loss": 3.8230936784291787, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040924774322968906, + "loss": 1.4842, + "theoretical_loss": 3.823010762908953, + "tokens_seen": 625999872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092276830491474, + "loss": 1.3199, + "theoretical_loss": 3.822927869607664, + "tokens_seen": 626130944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092076228686058, + "loss": 1.2685, + "theoretical_loss": 3.822844998514711, + "tokens_seen": 626262016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040918756268806415, + "loss": 1.5454, + "theoretical_loss": 3.8227621496194946, + "tokens_seen": 626393088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040916750250752257, + "loss": 1.406, + "theoretical_loss": 3.822679322911429, + "tokens_seen": 626524160 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409147442326981, + "loss": 1.1559, + "theoretical_loss": 3.82259651837993, + "tokens_seen": 626655232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040912738214643934, + "loss": 1.356, + "theoretical_loss": 3.8225137360144252, + "tokens_seen": 626786304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091073219658977, + "loss": 1.4281, + "theoretical_loss": 3.8224309758043464, + "tokens_seen": 626917376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090872617853561, + "loss": 1.2494, + "theoretical_loss": 3.822348237739134, + "tokens_seen": 627048448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090672016048145, + "loss": 1.3822, + "theoretical_loss": 3.822265521808236, + "tokens_seen": 627179520 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040904714142427285, + "loss": 1.4783, + "theoretical_loss": 3.8221828280011065, + "tokens_seen": 627310592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090270812437312, + "loss": 1.3068, + "theoretical_loss": 3.822100156307208, + "tokens_seen": 627441664 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.359665185213089, + "objective/train/avg_token_score": 0.019122768193483353, + "objective/train/docs_used": 364826, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3461437225341797, + "objective/train/original_loss": 3.9998061656951904, + "objective/train/theoretical_loss": 3.8220588287494284, + "objective/train/tokens_used": 627507200, + "objective/train/unlikelihood_loss": 0.16339793801307678, + "theoretical_loss": 3.8220588287494284, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040900702106318957, + "loss": 1.3026, + "theoretical_loss": 3.8220175067160085, + "tokens_seen": 627572736 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408986960882648, + "loss": 1.4798, + "theoretical_loss": 3.821934879216985, + "tokens_seen": 627703808 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040896690070210635, + "loss": 1.3787, + "theoretical_loss": 3.821852273799621, + "tokens_seen": 627834880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089468405215647, + "loss": 1.329, + "theoretical_loss": 3.821769690453407, + "tokens_seen": 627965952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089267803410231, + "loss": 1.3329, + "theoretical_loss": 3.8216871291678416, + "tokens_seen": 628097024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089067201604815, + "loss": 1.4236, + "theoretical_loss": 3.821604589932429, + "tokens_seen": 628228096 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040888665997993985, + "loss": 1.3524, + "theoretical_loss": 3.8215220727366823, + "tokens_seen": 628359168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088665997993982, + "loss": 1.4028, + "theoretical_loss": 3.82143957757012, + "tokens_seen": 628490240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088465396188566, + "loss": 1.4714, + "theoretical_loss": 3.8213571044222694, + "tokens_seen": 628621312 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408826479438315, + "loss": 1.3119, + "theoretical_loss": 3.8212746532826642, + "tokens_seen": 628752384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040880641925777336, + "loss": 1.3945, + "theoretical_loss": 3.8211922241408454, + "tokens_seen": 628883456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087863590772317, + "loss": 1.2536, + "theoretical_loss": 3.8211098169863607, + "tokens_seen": 629014528 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4605938494205475, + "objective/train/avg_token_score": 0.018184412270784378, + "objective/train/docs_used": 365375, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5830137729644775, + "objective/train/original_loss": 3.831573724746704, + "objective/train/theoretical_loss": 3.8210274318087656, + "objective/train/tokens_used": 629145600, + "objective/train/unlikelihood_loss": 0.12006247788667679, + "theoretical_loss": 3.8210274318087656, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087662988966901, + "loss": 1.4189, + "theoretical_loss": 3.8210274318087656, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040874623871614844, + "loss": 1.4326, + "theoretical_loss": 3.8209450685976227, + "tokens_seen": 629276672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040872617853560686, + "loss": 1.5258, + "theoretical_loss": 3.820862727342501, + "tokens_seen": 629407744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087061183550652, + "loss": 1.2155, + "theoretical_loss": 3.8207804080329772, + "tokens_seen": 629538816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086860581745236, + "loss": 1.3935, + "theoretical_loss": 3.8206981106586357, + "tokens_seen": 629669888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040866599799398195, + "loss": 1.4249, + "theoretical_loss": 3.8206158352090664, + "tokens_seen": 629800960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040864593781344036, + "loss": 1.4532, + "theoretical_loss": 3.8205335816738684, + "tokens_seen": 629932032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086258776328987, + "loss": 1.4963, + "theoretical_loss": 3.820451350042646, + "tokens_seen": 630063104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086058174523571, + "loss": 1.3688, + "theoretical_loss": 3.8203691403050106, + "tokens_seen": 630194176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040858575727181545, + "loss": 1.2832, + "theoretical_loss": 3.8202869524505827, + "tokens_seen": 630325248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085656970912738, + "loss": 1.5181, + "theoretical_loss": 3.8202047864689885, + "tokens_seen": 630456320 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040854563691073223, + "loss": 1.4243, + "theoretical_loss": 3.820122642349861, + "tokens_seen": 630587392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085255767301906, + "loss": 1.4029, + "theoretical_loss": 3.8200405200828405, + "tokens_seen": 630718464 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2529325485229492, + "objective/train/avg_token_score": 0.023288333788514137, + "objective/train/docs_used": 366612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9088571071624756, + "objective/train/original_loss": 4.29040002822876, + "objective/train/theoretical_loss": 3.8199994671406348, + "objective/train/tokens_used": 630784000, + "objective/train/unlikelihood_loss": 0.16850118339061737, + "theoretical_loss": 3.8199994671406348, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040850551654964895, + "loss": 1.3071, + "theoretical_loss": 3.819958419657574, + "tokens_seen": 630849536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084854563691073, + "loss": 1.4315, + "theoretical_loss": 3.819876341063718, + "tokens_seen": 630980608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040846539618856573, + "loss": 1.2791, + "theoretical_loss": 3.8197942842909325, + "tokens_seen": 631111680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084453360080241, + "loss": 1.3536, + "theoretical_loss": 3.8197122493288864, + "tokens_seen": 631242752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040842527582748246, + "loss": 1.3218, + "theoretical_loss": 3.8196302361672556, + "tokens_seen": 631373824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084052156469408, + "loss": 1.4322, + "theoretical_loss": 3.8195482447957225, + "tokens_seen": 631504896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083851554663992, + "loss": 1.5428, + "theoretical_loss": 3.8194662752039776, + "tokens_seen": 631635968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083650952858576, + "loss": 1.5233, + "theoretical_loss": 3.8193843273817167, + "tokens_seen": 631767040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040834503510531596, + "loss": 1.3592, + "theoretical_loss": 3.8193024013186445, + "tokens_seen": 631898112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083249749247743, + "loss": 1.3766, + "theoretical_loss": 3.819220497004471, + "tokens_seen": 632029184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083049147442327, + "loss": 1.4008, + "theoretical_loss": 3.8191386144289137, + "tokens_seen": 632160256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082848545636911, + "loss": 1.4924, + "theoretical_loss": 3.819056753581699, + "tokens_seen": 632291328 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4759897291660309, + "objective/train/avg_token_score": 0.017946870997548103, + "objective/train/docs_used": 367323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5524520874023438, + "objective/train/original_loss": 3.585705041885376, + "objective/train/theoretical_loss": 3.818974914452557, + "objective/train/tokens_used": 632422400, + "objective/train/unlikelihood_loss": 0.13466675579547882, + "theoretical_loss": 3.818974914452557, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040826479438314946, + "loss": 1.3453, + "theoretical_loss": 3.818974914452557, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082447342026078, + "loss": 1.3149, + "theoretical_loss": 3.8188930970312267, + "tokens_seen": 632553472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082246740220662, + "loss": 1.3998, + "theoretical_loss": 3.8188113013074547, + "tokens_seen": 632684544 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040820461384152455, + "loss": 1.2335, + "theoretical_loss": 3.8187295272709934, + "tokens_seen": 632815616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040818455366098297, + "loss": 1.1875, + "theoretical_loss": 3.818647774911601, + "tokens_seen": 632946688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040816449348044133, + "loss": 1.2951, + "theoretical_loss": 3.818566044219046, + "tokens_seen": 633077760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081444332998997, + "loss": 1.4905, + "theoretical_loss": 3.8184843351831006, + "tokens_seen": 633208832 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040812437311935805, + "loss": 1.2673, + "theoretical_loss": 3.8184026477935458, + "tokens_seen": 633339904 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040810431293881647, + "loss": 1.338, + "theoretical_loss": 3.818320982040169, + "tokens_seen": 633470976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040808425275827483, + "loss": 1.708, + "theoretical_loss": 3.8182393379127655, + "tokens_seen": 633602048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080641925777332, + "loss": 1.5884, + "theoretical_loss": 3.8181577154011346, + "tokens_seen": 633733120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804413239719156, + "loss": 1.4255, + "theoretical_loss": 3.818076114495086, + "tokens_seen": 633864192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080240722166499, + "loss": 1.5113, + "theoretical_loss": 3.817994535184434, + "tokens_seen": 633995264 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.5092253088951111, + "objective/train/avg_token_score": 0.017480170354247093, + "objective/train/docs_used": 367883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8412432670593262, + "objective/train/original_loss": 3.7791481018066406, + "objective/train/theoretical_loss": 3.8179537536242005, + "objective/train/tokens_used": 634060800, + "objective/train/unlikelihood_loss": 0.15870891511440277, + "theoretical_loss": 3.8179537536242005, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040800401203610834, + "loss": 1.3694, + "theoretical_loss": 3.8179129774590006, + "tokens_seen": 634126336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079839518555667, + "loss": 1.5419, + "theoretical_loss": 3.8178314413086154, + "tokens_seen": 634257408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040796389167502506, + "loss": 1.4045, + "theoretical_loss": 3.817749926723114, + "tokens_seen": 634388480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079438314944834, + "loss": 1.3526, + "theoretical_loss": 3.8176684336923383, + "tokens_seen": 634519552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040792377131394184, + "loss": 1.4804, + "theoretical_loss": 3.8175869622061382, + "tokens_seen": 634650624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079037111334002, + "loss": 1.5129, + "theoretical_loss": 3.817505512254371, + "tokens_seen": 634781696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040788365095285856, + "loss": 1.4568, + "theoretical_loss": 3.817424083826899, + "tokens_seen": 634912768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078635907723169, + "loss": 1.3891, + "theoretical_loss": 3.8173426769135927, + "tokens_seen": 635043840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078435305917753, + "loss": 1.3367, + "theoretical_loss": 3.8172612915043294, + "tokens_seen": 635174912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078234704112337, + "loss": 1.4437, + "theoretical_loss": 3.8171799275889926, + "tokens_seen": 635305984 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040780341023069207, + "loss": 1.3874, + "theoretical_loss": 3.8170985851574732, + "tokens_seen": 635437056 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040778335005015043, + "loss": 1.3605, + "theoretical_loss": 3.8170172641996682, + "tokens_seen": 635568128 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.39595553278923035, + "objective/train/avg_token_score": 0.020078817382454872, + "objective/train/docs_used": 369288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3277698755264282, + "objective/train/original_loss": 3.681797742843628, + "objective/train/theoretical_loss": 3.8169359647054835, + "objective/train/tokens_used": 635699200, + "objective/train/unlikelihood_loss": 0.16572554409503937, + "theoretical_loss": 3.8169359647054835, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077632898696088, + "loss": 1.3921, + "theoretical_loss": 3.8169359647054835, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077432296890672, + "loss": 1.2567, + "theoretical_loss": 3.8168546866648283, + "tokens_seen": 635830272 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040772316950852557, + "loss": 1.3112, + "theoretical_loss": 3.816773430067622, + "tokens_seen": 635961344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040770310932798393, + "loss": 1.5763, + "theoretical_loss": 3.816692194903789, + "tokens_seen": 636092416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076830491474423, + "loss": 1.3896, + "theoretical_loss": 3.816610981163261, + "tokens_seen": 636223488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076629889669007, + "loss": 1.4929, + "theoretical_loss": 3.8165297888359766, + "tokens_seen": 636354560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076429287863591, + "loss": 1.5615, + "theoretical_loss": 3.816448617911881, + "tokens_seen": 636485632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076228686058175, + "loss": 1.5079, + "theoretical_loss": 3.816367468380926, + "tokens_seen": 636616704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040760280842527585, + "loss": 1.2841, + "theoretical_loss": 3.8162863402330713, + "tokens_seen": 636747776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075827482447342, + "loss": 1.4782, + "theoretical_loss": 3.816205233458281, + "tokens_seen": 636878848 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040756268806419263, + "loss": 1.1803, + "theoretical_loss": 3.816124148046529, + "tokens_seen": 637009920 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407542627883651, + "loss": 1.5352, + "theoretical_loss": 3.8160430839877937, + "tokens_seen": 637140992 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040752256770310935, + "loss": 1.3779, + "theoretical_loss": 3.815962041272061, + "tokens_seen": 637272064 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.5428274869918823, + "objective/train/avg_token_score": 0.006741791497915983, + "objective/train/docs_used": 369983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.891964077949524, + "objective/train/original_loss": 3.4886183738708496, + "objective/train/theoretical_loss": 3.815921527914693, + "objective/train/tokens_used": 637337600, + "objective/train/unlikelihood_loss": 0.1536000370979309, + "theoretical_loss": 3.815921527914693, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075025075225677, + "loss": 1.4306, + "theoretical_loss": 3.815881019889324, + "tokens_seen": 637403136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040748244734202613, + "loss": 1.4211, + "theoretical_loss": 3.8158000198295814, + "tokens_seen": 637534208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074623871614845, + "loss": 1.4194, + "theoretical_loss": 3.81571904108284, + "tokens_seen": 637665280 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040744232698094286, + "loss": 1.28, + "theoretical_loss": 3.8156380836391124, + "tokens_seen": 637796352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074222668004012, + "loss": 1.4144, + "theoretical_loss": 3.815557147488418, + "tokens_seen": 637927424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074022066198596, + "loss": 1.4748, + "theoretical_loss": 3.815476232620784, + "tokens_seen": 638058496 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407382146439318, + "loss": 1.4531, + "theoretical_loss": 3.8153953390262427, + "tokens_seen": 638189568 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040736208625877636, + "loss": 1.3289, + "theoretical_loss": 3.8153144666948338, + "tokens_seen": 638320640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073420260782347, + "loss": 1.4632, + "theoretical_loss": 3.815233615616605, + "tokens_seen": 638451712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073219658976931, + "loss": 1.5634, + "theoretical_loss": 3.8151527857816077, + "tokens_seen": 638582784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073019057171515, + "loss": 1.483, + "theoretical_loss": 3.815071977179903, + "tokens_seen": 638713856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040728184553660986, + "loss": 1.4585, + "theoretical_loss": 3.814991189801557, + "tokens_seen": 638844928 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2507942318916321, + "objective/train/avg_token_score": 0.031877949833869934, + "objective/train/docs_used": 370882, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.902974545955658, + "objective/train/original_loss": 4.440757751464844, + "objective/train/theoretical_loss": 3.8149104236366433, + "objective/train/tokens_used": 638976000, + "objective/train/unlikelihood_loss": 0.10956820845603943, + "theoretical_loss": 3.8149104236366433, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072617853560682, + "loss": 1.54, + "theoretical_loss": 3.8149104236366433, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072417251755266, + "loss": 1.4755, + "theoretical_loss": 3.814829678675242, + "tokens_seen": 639107072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040722166499498495, + "loss": 1.2833, + "theoretical_loss": 3.814748954907439, + "tokens_seen": 639238144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040720160481444337, + "loss": 1.3224, + "theoretical_loss": 3.8146682523233286, + "tokens_seen": 639369216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040718154463390173, + "loss": 1.5405, + "theoretical_loss": 3.8145875709130093, + "tokens_seen": 639500288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071614844533601, + "loss": 1.4171, + "theoretical_loss": 3.814506910666589, + "tokens_seen": 639631360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040714142427281845, + "loss": 1.5556, + "theoretical_loss": 3.81442627157418, + "tokens_seen": 639762432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040712136409227687, + "loss": 1.5124, + "theoretical_loss": 3.8143456536259035, + "tokens_seen": 639893504 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040710130391173523, + "loss": 1.4711, + "theoretical_loss": 3.8142650568118848, + "tokens_seen": 640024576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070812437311936, + "loss": 1.4587, + "theoretical_loss": 3.814184481122257, + "tokens_seen": 640155648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040706118355065196, + "loss": 1.4772, + "theoretical_loss": 3.814103926547161, + "tokens_seen": 640286720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070411233701103, + "loss": 1.5276, + "theoretical_loss": 3.8140233930767424, + "tokens_seen": 640417792 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040702106318956874, + "loss": 1.4636, + "theoretical_loss": 3.8139428807011537, + "tokens_seen": 640548864 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2876954972743988, + "objective/train/avg_token_score": 0.005228789057582617, + "objective/train/docs_used": 371543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9728174805641174, + "objective/train/original_loss": 3.6427764892578125, + "objective/train/theoretical_loss": 3.8139026324208456, + "objective/train/tokens_used": 640614400, + "objective/train/unlikelihood_loss": 0.19972942769527435, + "theoretical_loss": 3.8139026324208456, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070010030090271, + "loss": 1.3805, + "theoretical_loss": 3.8138623894105557, + "tokens_seen": 640679936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040698094282848546, + "loss": 1.5623, + "theoretical_loss": 3.813781919195114, + "tokens_seen": 640811008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069608826479438, + "loss": 1.5725, + "theoretical_loss": 3.813701470045001, + "tokens_seen": 640942080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040694082246740224, + "loss": 1.4707, + "theoretical_loss": 3.8136210419503977, + "tokens_seen": 641073152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069207622868606, + "loss": 1.4157, + "theoretical_loss": 3.8135406349014884, + "tokens_seen": 641204224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040690070210631896, + "loss": 1.4061, + "theoretical_loss": 3.8134602488884664, + "tokens_seen": 641335296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068806419257773, + "loss": 1.3694, + "theoretical_loss": 3.813379883901531, + "tokens_seen": 641466368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068605817452357, + "loss": 1.4924, + "theoretical_loss": 3.8132995399308873, + "tokens_seen": 641597440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068405215646941, + "loss": 1.5049, + "theoretical_loss": 3.8132192169667483, + "tokens_seen": 641728512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040682046138415247, + "loss": 1.2999, + "theoretical_loss": 3.813138914999332, + "tokens_seen": 641859584 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040680040120361083, + "loss": 1.3115, + "theoretical_loss": 3.813058634018865, + "tokens_seen": 641990656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067803410230692, + "loss": 1.3317, + "theoretical_loss": 3.812978374015578, + "tokens_seen": 642121728 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.43664467334747314, + "objective/train/avg_token_score": 0.01415205281227827, + "objective/train/docs_used": 373013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5176647901535034, + "objective/train/original_loss": 3.9611873626708984, + "objective/train/theoretical_loss": 3.8128981349797098, + "objective/train/tokens_used": 642252800, + "objective/train/unlikelihood_loss": 0.1576123982667923, + "theoretical_loss": 3.8128981349797098, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067602808425276, + "loss": 1.6215, + "theoretical_loss": 3.8128981349797098, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040674022066198597, + "loss": 1.4195, + "theoretical_loss": 3.8128179169015057, + "tokens_seen": 642383872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040672016048144433, + "loss": 1.4247, + "theoretical_loss": 3.8127377197712176, + "tokens_seen": 642514944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067001003009027, + "loss": 1.4622, + "theoretical_loss": 3.812657543579103, + "tokens_seen": 642646016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066800401203611, + "loss": 1.3565, + "theoretical_loss": 3.812577388315426, + "tokens_seen": 642777088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040665997993981947, + "loss": 1.5205, + "theoretical_loss": 3.812497253970459, + "tokens_seen": 642908160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040663991975927783, + "loss": 1.5612, + "theoretical_loss": 3.812417140534478, + "tokens_seen": 643039232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066198595787362, + "loss": 1.3668, + "theoretical_loss": 3.8123370479977687, + "tokens_seen": 643170304 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040659979939819456, + "loss": 1.4831, + "theoretical_loss": 3.8122569763506204, + "tokens_seen": 643301376 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406579739217653, + "loss": 1.5363, + "theoretical_loss": 3.812176925583331, + "tokens_seen": 643432448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040655967903711134, + "loss": 1.3226, + "theoretical_loss": 3.8120968956862034, + "tokens_seen": 643563520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065396188565697, + "loss": 1.4465, + "theoretical_loss": 3.8120168866495483, + "tokens_seen": 643694592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040651955867602806, + "loss": 1.5168, + "theoretical_loss": 3.8119368984636814, + "tokens_seen": 643825664 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.45222383737564087, + "objective/train/avg_token_score": 0.015421704389154911, + "objective/train/docs_used": 373533, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5024516582489014, + "objective/train/original_loss": 3.635314464569092, + "objective/train/theoretical_loss": 3.8118969121867696, + "objective/train/tokens_used": 643891200, + "objective/train/unlikelihood_loss": 0.1828690469264984, + "theoretical_loss": 3.8118969121867696, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064994984954865, + "loss": 1.2832, + "theoretical_loss": 3.8118569311189265, + "tokens_seen": 643956736 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040647943831494484, + "loss": 1.453, + "theoretical_loss": 3.811776984605612, + "tokens_seen": 644087808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064593781344032, + "loss": 1.4285, + "theoretical_loss": 3.811697058914075, + "tokens_seen": 644218880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040643931795386157, + "loss": 1.4884, + "theoretical_loss": 3.811617154034657, + "tokens_seen": 644349952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040641925777331993, + "loss": 1.4966, + "theoretical_loss": 3.8115372699577073, + "tokens_seen": 644481024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040639919759277834, + "loss": 1.4494, + "theoretical_loss": 3.8114574066735805, + "tokens_seen": 644612096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063791374122367, + "loss": 1.4307, + "theoretical_loss": 3.8113775641726386, + "tokens_seen": 644743168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040635907723169507, + "loss": 1.3986, + "theoretical_loss": 3.81129774244525, + "tokens_seen": 644874240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040633901705115343, + "loss": 1.6454, + "theoretical_loss": 3.8112179414817877, + "tokens_seen": 645005312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040631895687061185, + "loss": 1.6522, + "theoretical_loss": 3.8111381612726345, + "tokens_seen": 645136384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062988966900702, + "loss": 1.6055, + "theoretical_loss": 3.811058401808177, + "tokens_seen": 645267456 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040627883650952857, + "loss": 1.479, + "theoretical_loss": 3.8109786630788083, + "tokens_seen": 645398528 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.4299853444099426, + "objective/train/avg_token_score": 0.014909038320183754, + "objective/train/docs_used": 374553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5298352241516113, + "objective/train/original_loss": 4.025496006011963, + "objective/train/theoretical_loss": 3.8108989450749293, + "objective/train/tokens_used": 645529600, + "objective/train/unlikelihood_loss": 0.1589026004076004, + "theoretical_loss": 3.8108989450749293, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040625877632898693, + "loss": 1.5565, + "theoretical_loss": 3.8108989450749293, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062387161484453, + "loss": 1.3728, + "theoretical_loss": 3.810819247786946, + "tokens_seen": 645660672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062186559679037, + "loss": 1.518, + "theoretical_loss": 3.8107395712052714, + "tokens_seen": 645791744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061985957873621, + "loss": 1.3697, + "theoretical_loss": 3.8106599153203256, + "tokens_seen": 645922816 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040617853560682044, + "loss": 1.53, + "theoretical_loss": 3.810580280122533, + "tokens_seen": 646053888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061584754262788, + "loss": 1.4548, + "theoretical_loss": 3.8105006656023264, + "tokens_seen": 646184960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040613841524573727, + "loss": 1.4705, + "theoretical_loss": 3.8104210717501434, + "tokens_seen": 646316032 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040611835506519563, + "loss": 1.5978, + "theoretical_loss": 3.8103414985564292, + "tokens_seen": 646447104 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406098294884654, + "loss": 1.3603, + "theoretical_loss": 3.810261946011636, + "tokens_seen": 646578176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040607823470411236, + "loss": 1.4798, + "theoretical_loss": 3.810182414106219, + "tokens_seen": 646709248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060581745235707, + "loss": 1.4606, + "theoretical_loss": 3.8101029028306437, + "tokens_seen": 646840320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040603811434302914, + "loss": 1.5778, + "theoretical_loss": 3.8100234121753798, + "tokens_seen": 646971392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060180541624875, + "loss": 1.5656, + "theoretical_loss": 3.8099439421309036, + "tokens_seen": 647102464 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.45576736330986023, + "objective/train/avg_token_score": 0.028985600918531418, + "objective/train/docs_used": 375146, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.656118392944336, + "objective/train/original_loss": 3.9800736904144287, + "objective/train/theoretical_loss": 3.8099042148347366, + "objective/train/tokens_used": 647168000, + "objective/train/unlikelihood_loss": 0.11932669579982758, + "theoretical_loss": 3.8099042148347366, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040599799398194586, + "loss": 1.4929, + "theoretical_loss": 3.809864492687698, + "tokens_seen": 647233536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059779338014042, + "loss": 1.5617, + "theoretical_loss": 3.8097850638362516, + "tokens_seen": 647364608 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040595787362086264, + "loss": 1.2915, + "theoretical_loss": 3.8097056555670608, + "tokens_seen": 647495680 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405937813440321, + "loss": 1.4292, + "theoretical_loss": 3.8096262678706267, + "tokens_seen": 647626752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040591775325977936, + "loss": 1.5357, + "theoretical_loss": 3.809546900737457, + "tokens_seen": 647757824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058976930792377, + "loss": 1.3051, + "theoretical_loss": 3.809467554158067, + "tokens_seen": 647888896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058776328986961, + "loss": 1.4298, + "theoretical_loss": 3.8093882281229767, + "tokens_seen": 648019968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058575727181545, + "loss": 1.3352, + "theoretical_loss": 3.809308922622713, + "tokens_seen": 648151040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040583751253761287, + "loss": 1.512, + "theoretical_loss": 3.8092296376478094, + "tokens_seen": 648282112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040581745235707123, + "loss": 1.4593, + "theoretical_loss": 3.809150373188805, + "tokens_seen": 648413184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057973921765296, + "loss": 1.3488, + "theoretical_loss": 3.8090711292362456, + "tokens_seen": 648544256 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405777331995988, + "loss": 1.4565, + "theoretical_loss": 3.808991905780683, + "tokens_seen": 648675328 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.455278605222702, + "objective/train/avg_token_score": 0.015043376013636589, + "objective/train/docs_used": 375860, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5879684686660767, + "objective/train/original_loss": 3.877953290939331, + "objective/train/theoretical_loss": 3.8089127028126764, + "objective/train/tokens_used": 648806400, + "objective/train/unlikelihood_loss": 0.21014168858528137, + "theoretical_loss": 3.8089127028126764, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040575727181544637, + "loss": 1.4726, + "theoretical_loss": 3.8089127028126764, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040573721163490473, + "loss": 1.4837, + "theoretical_loss": 3.8088335203227897, + "tokens_seen": 648937472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057171514543631, + "loss": 1.4837, + "theoretical_loss": 3.8087543583015933, + "tokens_seen": 649068544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040569709127382146, + "loss": 1.338, + "theoretical_loss": 3.808675216739665, + "tokens_seen": 649199616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056770310932799, + "loss": 1.4928, + "theoretical_loss": 3.8085960956275873, + "tokens_seen": 649330688 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040565697091273824, + "loss": 1.5526, + "theoretical_loss": 3.8085169949559505, + "tokens_seen": 649461760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056369107321966, + "loss": 1.4267, + "theoretical_loss": 3.8084379147153493, + "tokens_seen": 649592832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040561685055165496, + "loss": 1.3143, + "theoretical_loss": 3.8083588548963867, + "tokens_seen": 649723904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055967903711134, + "loss": 1.4323, + "theoretical_loss": 3.808279815489671, + "tokens_seen": 649854976 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040557673019057174, + "loss": 1.2942, + "theoretical_loss": 3.808200796485815, + "tokens_seen": 649986048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055566700100301, + "loss": 1.4689, + "theoretical_loss": 3.8081217978754407, + "tokens_seen": 650117120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040553660982948846, + "loss": 1.6256, + "theoretical_loss": 3.8080428196491747, + "tokens_seen": 650248192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055165496489469, + "loss": 1.3445, + "theoretical_loss": 3.8079638617976492, + "tokens_seen": 650379264 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.4335899353027344, + "objective/train/avg_token_score": 0.011522009037435055, + "objective/train/docs_used": 377374, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4454277753829956, + "objective/train/original_loss": 3.9384045600891113, + "objective/train/theoretical_loss": 3.8079243905094886, + "objective/train/tokens_used": 650444800, + "objective/train/unlikelihood_loss": 0.13931621611118317, + "theoretical_loss": 3.8079243905094886, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040549648946840524, + "loss": 1.4248, + "theoretical_loss": 3.8078849243115043, + "tokens_seen": 650510336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054764292878636, + "loss": 1.3241, + "theoretical_loss": 3.8078060071813846, + "tokens_seen": 650641408 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040545636910732197, + "loss": 1.6134, + "theoretical_loss": 3.807727110397942, + "tokens_seen": 650772480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040543630892678033, + "loss": 1.505, + "theoretical_loss": 3.807648233951835, + "tokens_seen": 650903552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040541624874623874, + "loss": 1.5242, + "theoretical_loss": 3.807569377833726, + "tokens_seen": 651034624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053961885656971, + "loss": 1.3798, + "theoretical_loss": 3.8074905420342864, + "tokens_seen": 651165696 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040537612838515547, + "loss": 1.5358, + "theoretical_loss": 3.807411726544191, + "tokens_seen": 651296768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040535606820461383, + "loss": 1.6171, + "theoretical_loss": 3.8073329313541233, + "tokens_seen": 651427840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040533600802407225, + "loss": 1.5088, + "theoretical_loss": 3.807254156454772, + "tokens_seen": 651558912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053159478435306, + "loss": 1.504, + "theoretical_loss": 3.8071754018368305, + "tokens_seen": 651689984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040529588766298897, + "loss": 1.4844, + "theoretical_loss": 3.8070966674910007, + "tokens_seen": 651821056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040527582748244733, + "loss": 1.308, + "theoretical_loss": 3.807017953407989, + "tokens_seen": 651952128 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.30968964099884033, + "objective/train/avg_token_score": 0.01105769444257021, + "objective/train/docs_used": 377873, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0822099447250366, + "objective/train/original_loss": 4.107913494110107, + "objective/train/theoretical_loss": 3.8069392595785083, + "objective/train/tokens_used": 652083200, + "objective/train/unlikelihood_loss": 0.13217970728874207, + "theoretical_loss": 3.8069392595785083, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052557673019057, + "loss": 1.5052, + "theoretical_loss": 3.8069392595785083, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052357071213641, + "loss": 1.4459, + "theoretical_loss": 3.8068605859932783, + "tokens_seen": 652214272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052156469408225, + "loss": 1.5091, + "theoretical_loss": 3.806781932643024, + "tokens_seen": 652345344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040519558676028084, + "loss": 1.3071, + "theoretical_loss": 3.8067032995184764, + "tokens_seen": 652476416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051755265797392, + "loss": 1.4541, + "theoretical_loss": 3.806624686610374, + "tokens_seen": 652607488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051554663991976, + "loss": 1.3498, + "theoretical_loss": 3.8065460939094597, + "tokens_seen": 652738560 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405135406218656, + "loss": 1.5227, + "theoretical_loss": 3.806467521406484, + "tokens_seen": 652869632 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040511534603811434, + "loss": 1.5705, + "theoretical_loss": 3.8063889690922013, + "tokens_seen": 653000704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050952858575727, + "loss": 1.6055, + "theoretical_loss": 3.806310436957374, + "tokens_seen": 653131776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040507522567703107, + "loss": 1.3488, + "theoretical_loss": 3.8062319249927707, + "tokens_seen": 653262848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050551654964895, + "loss": 1.4862, + "theoretical_loss": 3.806153433189165, + "tokens_seen": 653393920 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040503510531594784, + "loss": 1.5408, + "theoretical_loss": 3.8060749615373375, + "tokens_seen": 653524992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050150451354062, + "loss": 1.4192, + "theoretical_loss": 3.8059965100280735, + "tokens_seen": 653656064 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.4617546498775482, + "objective/train/avg_token_score": 0.007299356162548065, + "objective/train/docs_used": 379015, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.598456621170044, + "objective/train/original_loss": 3.781627655029297, + "objective/train/theoretical_loss": 3.8059572918240248, + "objective/train/tokens_used": 653721600, + "objective/train/unlikelihood_loss": 0.2492816299200058, + "theoretical_loss": 3.8059572918240248, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040499498495486457, + "loss": 1.5622, + "theoretical_loss": 3.8059180786521654, + "tokens_seen": 653787136 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404974924774323, + "loss": 1.5489, + "theoretical_loss": 3.805839667400412, + "tokens_seen": 653918208 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040495486459378135, + "loss": 1.4142, + "theoretical_loss": 3.8057612762636173, + "tokens_seen": 654049280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049348044132397, + "loss": 1.4089, + "theoretical_loss": 3.8056829052325916, + "tokens_seen": 654180352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040491474423269807, + "loss": 1.4654, + "theoretical_loss": 3.8056045542981516, + "tokens_seen": 654311424 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040489468405215643, + "loss": 1.4081, + "theoretical_loss": 3.8055262234511193, + "tokens_seen": 654442496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040487462387161485, + "loss": 1.4031, + "theoretical_loss": 3.805447912682324, + "tokens_seen": 654573568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048545636910732, + "loss": 1.3817, + "theoretical_loss": 3.8053696219825994, + "tokens_seen": 654704640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048345035105316, + "loss": 1.3633, + "theoretical_loss": 3.8052913513427864, + "tokens_seen": 654835712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040481444332998994, + "loss": 1.4964, + "theoretical_loss": 3.805213100753731, + "tokens_seen": 654966784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040479438314944835, + "loss": 1.3457, + "theoretical_loss": 3.8051348702062864, + "tokens_seen": 655097856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047743229689067, + "loss": 1.3372, + "theoretical_loss": 3.805056659691311, + "tokens_seen": 655228928 + }, + { + "debugging/length": 515.6, + "debugging/num_segments": 5, + "epoch": 0.2, + "objective/train/aligned_token_count": 0.3613147735595703, + "objective/train/avg_token_score": 0.006067417562007904, + "objective/train/docs_used": 379813, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.339416742324829, + "objective/train/original_loss": 3.8491370677948, + "objective/train/theoretical_loss": 3.804978469199669, + "objective/train/tokens_used": 655360000, + "objective/train/unlikelihood_loss": 0.20670151710510254, + "theoretical_loss": 3.804978469199669, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047542627883651, + "loss": 1.4639, + "theoretical_loss": 3.804978469199669, + "tokens_seen": 655360000 + } + ], + "max_steps": 25177, + "num_train_epochs": 9223372036854775807, + "total_flos": 3.3445380096e+17, + "trial_name": null, + "trial_params": null +}